1mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7 let artifacts = home.join("runs").join(&state.id).join("artifacts");
8 if !artifacts.try_exists()? {
9 return Ok(false);
10 }
11 ensure!(
12 plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13 "noncanonical artifacts"
14 );
15 for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16 ensure!(i < 4096, "eviction record inspection limit exceeded");
17 let name = entry?.file_name();
18 let name = name.to_string_lossy();
19 if name.starts_with("eviction-") && name.ends_with(".json") {
20 return Ok(true);
21 }
22 }
23 Ok(false)
24}
25
26pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29 copies::restore_execution_seats(home, state).await
30}
31
32pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34 crate::fold_evidence::validate_snapshot(evidence).await?;
35 Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40 crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43pub fn seat_inspections(home: &Path, id: &str) -> Result<Vec<String>> {
45 ensure!(crate::run::is_run_id(id), "invalid run id");
46 copies::seat_inspections(home, id)
47}
48
49pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
51 copies::seat_evidence(home, id).await
52}
53
54pub async fn seat_heads(
56 home: &Path,
57 id: &str,
58) -> Result<std::collections::BTreeMap<String, String>> {
59 copies::seat_heads(home, id).await
60}
61
62pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
64 copies::seat_diff(home, id, head).await
65}
66
67pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
69 copies::restore_seat(home, id, head, destination).await
70}
71
72use crate::{
73 git,
74 proc::Quiet as _,
75 run::{Liveness, RunState},
76};
77use anyhow::{Context, Result, ensure};
78use jiff::Timestamp;
79use serde::{Deserialize, Serialize};
80use std::{
81 fs::{File, OpenOptions},
82 path::{Path, PathBuf},
83};
84
85#[derive(Debug)]
89pub(crate) struct OutputLease {
90 _file: std::sync::Arc<OutputFile>,
91 _activity: bool,
92}
93
94#[derive(Debug)]
95struct OutputFile(File);
96
97impl std::ops::Deref for OutputFile {
98 type Target = File;
99 fn deref(&self) -> &File {
100 &self.0
101 }
102}
103
104#[cfg(unix)]
105impl Drop for OutputFile {
106 fn drop(&mut self) {
107 use std::os::fd::AsRawFd;
108 unsafe {
110 libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
111 }
112 }
113}
114
115impl OutputLease {
116 #[cfg(unix)]
117 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
118 use std::os::fd::AsRawFd;
119 self._file.as_raw_fd()
120 }
121
122 pub(crate) fn pre_clean(dir: &Path) -> Result<Self> {
124 let file = OpenOptions::new()
125 .create(true)
126 .truncate(false)
127 .read(true)
128 .write(true)
129 .open(dir.join("magi-pre-clean.lock"))?;
130 crate::pr_head::try_lock_file(&file)
131 .context("pre-clean already running; retry after descendants terminate")?;
132 Ok(Self {
133 _file: std::sync::Arc::new(OutputFile(file)),
134 _activity: false,
135 })
136 }
137
138 pub(crate) fn duplicate(&self) -> Self {
139 Self {
140 _file: std::sync::Arc::clone(&self._file),
141 _activity: false,
142 }
143 }
144
145 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
146 if !worktree.join(".git").is_file() {
147 return Ok(None);
148 }
149 Self::take(worktree, false).map(Some)
150 }
151 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
152 use std::hash::{Hash, Hasher};
153 let mut hash = std::collections::hash_map::DefaultHasher::new();
154 let mut normalized = cache.to_owned();
155 let mut missing = Vec::new();
156 while !normalized.exists() {
157 let Some(name) = normalized.file_name() else {
158 break;
159 };
160 missing.push(name.to_owned());
161 if !normalized.pop() {
162 break;
163 }
164 }
165 normalized = normalized.canonicalize().unwrap_or(normalized);
166 for part in missing.into_iter().rev() {
167 normalized.push(part);
168 }
169 normalized.hash(&mut hash);
170 let dir = home.join("cache-leases");
171 std::fs::create_dir_all(&dir)?;
172 Self::lock(
173 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
174 exclusive,
175 )
176 }
177 #[cfg(unix)]
181 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
182 Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
183 }
184
185 #[cfg(unix)]
186 pub(crate) fn admit_with(
187 home: &Path,
188 paths: &[PathBuf],
189 min_free: u64,
190 free_bytes: impl Fn(&Path) -> Result<u64>,
191 ) -> Result<Vec<Self>> {
192 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
193 if min_free == 0 {
194 return Ok(Vec::new());
195 }
196 const RESERVE: u64 = 1024 * 1024 * 1024;
197 std::fs::create_dir_all(home)?;
198 let dir = home.canonicalize()?.join("disk-admission");
199 std::fs::create_dir_all(&dir)?;
200 ensure!(
201 plain_dir(&dir) && dir.canonicalize()? == dir,
202 "disk admission directory is not canonical"
203 );
204 let mut devices = std::collections::BTreeSet::new();
205 let mut reservations = Vec::new();
206 for path in paths {
207 let probe = path
208 .ancestors()
209 .find(|p| p.exists())
210 .context("disk admission path missing")?;
211 let device = std::fs::metadata(probe)?.dev();
212 if !devices.insert(device) {
213 continue;
214 }
215 let open = |path: &Path| -> Result<File> {
216 Ok(OpenOptions::new()
217 .read(true)
218 .write(true)
219 .create(true)
220 .truncate(false)
221 .custom_flags(libc::O_NOFOLLOW)
222 .open(path)?)
223 };
224 let admission = Self {
228 _file: std::sync::Arc::new(OutputFile(open(&dir.join(format!("{device}.lock")))?)),
229 _activity: false,
230 };
231 let started = std::time::Instant::now();
232 loop {
233 match crate::pr_head::try_lock_file(&admission._file) {
234 Ok(()) => break,
235 Err(error)
236 if matches!(
237 error.kind(),
238 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
239 ) && started.elapsed() < std::time::Duration::from_secs(1) =>
240 {
241 std::thread::sleep(std::time::Duration::from_millis(5));
242 }
243 Err(error) => {
244 return Err(error)
245 .context("disk admission busy or unavailable; retry without dispatch");
246 }
247 }
248 }
249 let mut active = 0u64;
250 let mut available = None;
251 for slot in 0..1024 {
252 let file = dir.join(format!("{device}-{slot}.reserve"));
253 if !file.exists() {
254 if available.is_none() {
255 available = Some(file);
256 }
257 continue;
258 }
259 let file = open(&file)?;
260 match crate::pr_head::try_lock_file(&file) {
261 Ok(()) if available.is_none() => {
262 available = Some(dir.join(format!("{device}-{slot}.reserve")))
263 }
264 Ok(()) => {}
265 Err(_) => active = active.saturating_add(RESERVE),
266 }
267 }
268 let free = free_bytes(probe)?;
269 let required = min_free.saturating_add(active).saturating_add(RESERVE);
270 ensure!(
271 free >= required,
272 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
273 );
274 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
275 crate::pr_head::try_lock_file(&file)?;
276 reservations.push(Self {
277 _file: std::sync::Arc::new(OutputFile(file)),
278 _activity: false,
279 });
280 }
281 Ok(reservations)
282 }
283
284 #[cfg(not(unix))]
285 pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
286 Ok(Vec::new())
288 }
289
290 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
291 let out = std::process::Command::new("git")
292 .args(["rev-parse", "--absolute-git-dir"])
293 .current_dir(worktree)
294 .quiet()
295 .output()?;
296 ensure!(
297 out.status.success(),
298 "cannot identify output lease directory"
299 );
300 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
301 Self::lock(&dir.join("magi-output.lock"), exclusive)
302 }
303 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
304 let file = OpenOptions::new()
305 .create(true)
306 .truncate(false)
307 .read(true)
308 .write(true)
309 .open(path)
310 .with_context(|| format!("output lease: open {}", path.display()))?;
311 #[cfg(unix)]
312 {
313 use std::os::fd::AsRawFd;
314 let mode = if exclusive {
315 libc::LOCK_EX
316 } else {
317 libc::LOCK_SH
318 };
319 ensure!(
321 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
322 "output lease: outputs are in use or being reclaimed: {}",
323 path.display()
324 );
325 }
326 #[cfg(not(unix))]
327 let _ = exclusive;
328 if !exclusive {
329 file.set_len(1)?;
331 file.set_modified(std::time::SystemTime::now())
332 .with_context(|| format!("record output activity {}", path.display()))?;
333 }
334 Ok(Self {
335 _file: std::sync::Arc::new(OutputFile(file)),
336 _activity: !exclusive,
337 })
338 }
339}
340#[cfg(unix)]
341impl Drop for OutputLease {
342 fn drop(&mut self) {
343 if self._activity
344 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
345 {
346 let _ = self._file.set_len(2);
349 tracing::warn!("could not record output lease release activity: {error}");
350 }
351 }
352}
353
354#[derive(Default, Debug, Serialize, Deserialize)]
355#[serde(default)]
356pub(crate) struct Report {
357 pub checked_at: Option<Timestamp>,
358 pub limit_bytes: u64,
359 pub reclaim_budget_bytes: u64,
360 pub measured_bytes: u64,
362 pub freed_bytes: u64,
364 pub measured_allocated_file_bytes: Option<u64>,
365 pub proven_reclaimable_bytes: u64,
367 pub proven_reclaimable_allocated_bytes: Option<u64>,
370 pub protected_or_unproven_bytes: u64,
371 pub estimated_released_allocated_bytes: Option<u64>,
374 pub observed_home_free_change_bytes: Option<i64>,
377 pub retired: usize,
378 pub evicted_copies: usize,
379 pub source_scan: Option<serde_json::Value>,
380 pub removed_empty_roots: usize,
381 pub source_unlinked_logical_bytes: u64,
382 pub source_unlinked_allocated_bytes: Option<u64>,
383 pub source_evidence_added_logical_bytes: u64,
384 pub source_evidence_added_allocated_bytes: Option<u64>,
385 pub retained: Vec<String>,
386 pub failures: Vec<String>,
387}
388
389#[cfg(unix)]
390fn allocated_inventory(
391 root: &Path,
392 seen: &mut std::collections::BTreeSet<(u64, u64)>,
393) -> Result<u64> {
394 use std::os::unix::fs::MetadataExt;
395 ensure!(
396 plain_dir(root) && root.canonicalize()? == root,
397 "allocated inventory root is not canonical"
398 );
399 let mut dirs = vec![root.to_owned()];
400 let mut bytes = 0u64;
401 while let Some(dir) = dirs.pop() {
402 for entry in std::fs::read_dir(dir)? {
403 let path = entry?.path();
404 let metadata = std::fs::symlink_metadata(&path)?;
405 if metadata.is_dir() {
406 dirs.push(path);
407 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
408 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
409 }
410 }
411 }
412 Ok(bytes)
413}
414
415#[derive(Default)]
416struct ReleasedAllocation {
417 #[cfg(unix)]
418 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
419 #[cfg(unix)]
420 removed: std::collections::BTreeMap<(u64, u64), u64>,
421}
422
423impl ReleasedAllocation {
424 fn of(paths: &[PathBuf]) -> Result<Self> {
425 #[cfg(unix)]
426 {
427 use std::os::unix::fs::MetadataExt;
428 let mut files = std::collections::BTreeMap::new();
429 for path in paths {
430 let m = std::fs::symlink_metadata(path)?;
431 ensure!(m.is_file(), "allocation candidate is not regular");
432 files.insert(
433 path.clone(),
434 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
435 );
436 }
437 Ok(Self {
438 files,
439 removed: Default::default(),
440 })
441 }
442 #[cfg(not(unix))]
443 {
444 let _ = paths;
445 Ok(Self::default())
446 }
447 }
448 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
449 #[cfg(unix)]
450 {
451 let mut inodes = std::collections::BTreeMap::new();
452 for &(device, inode, links, blocks) in self.files.values() {
453 let entry = inodes
454 .entry((device, inode))
455 .or_insert((0u64, links, blocks));
456 entry.0 += 1;
457 }
458 Some(
459 inodes
460 .values()
461 .filter(|(count, links, _)| count == links)
462 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
463 )
464 }
465 #[cfg(not(unix))]
466 {
467 None
468 }
469 }
470
471 fn unlinked(&mut self, path: &Path) -> Option<u64> {
472 #[cfg(unix)]
473 {
474 let (device, inode, links, blocks) = self.files[path];
475 let removed = self.removed.entry((device, inode)).or_default();
476 *removed += 1;
477 Some(if *removed == links { blocks } else { 0 })
478 }
479 #[cfg(not(unix))]
480 {
481 let _ = path;
482 None
483 }
484 }
485}
486
487fn plain_dir(path: &Path) -> bool {
488 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
489}
490
491pub(crate) fn read_seat(path: &Path) -> bool {
492 path.file_name()
493 .and_then(|s| s.to_str())
494 .is_some_and(|name| {
495 ["review-", "judge-", "advisor-"].iter().any(|prefix| {
496 name.strip_prefix(prefix)
497 .is_some_and(|n| n.parse::<usize>().is_ok_and(|n| n > 0))
498 })
499 })
500}
501
502fn run_paths_missing(state: &RunState) -> Result<bool> {
505 for path in [state.worktree_root(), state.repo.clone()] {
506 match std::fs::metadata(&path) {
507 Ok(_) => {}
508 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
509 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
510 }
511 }
512 Ok(false)
513}
514
515pub(crate) async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
517 let root = state.worktree_root();
518 if !plain_dir(&root) {
519 return Ok(Vec::new());
520 }
521 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
522 registered_worktrees(state, &list)
523}
524
525fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
526 let root = state.worktree_root();
527 if !plain_dir(&root) {
528 return Ok(Vec::new());
529 }
530 let root = root.canonicalize()?;
531 let mut paths = Vec::new();
532 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
533 let path = PathBuf::from(path);
534 #[cfg(windows)]
535 let path = if plain_dir(&path) {
536 git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
537 } else {
538 continue;
539 };
540 if plain_dir(&path)
541 && path.parent() == Some(root.as_path())
542 && path.canonicalize()? == path
543 && path.join(".git").is_file()
544 {
545 paths.push(path);
546 }
547 }
548 Ok(paths)
549}
550
551async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
554 validate_cargo_target(worktree, false).await
555}
556
557async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
558 let target = worktree.join("target");
559 ensure!(plain_dir(&target), "target absent or not a plain directory");
560 ensure!(target.canonicalize()? == target, "target path changed");
561 ensure!(
562 git::git(worktree, &["ls-files", "--", "target"])
563 .await?
564 .is_empty(),
565 "tracked target contents"
566 );
567 ensure!(
568 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
569 .await?
570 .ok(),
571 "target is not ignored"
572 );
573 cache_tag(&target, allow_missing_tag)?;
574 let path = target.join(".rustc_info.json");
575 plain_file(&path, "rustc identity")?;
576 let info: serde_json::Value = serde_json::from_slice(
577 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
578 )
579 .with_context(|| format!("parse rustc identity {}", path.display()))?;
580 ensure!(
581 info.get("rustc_fingerprint").is_some(),
582 "unknown Cargo identity {}",
583 path.display()
584 );
585 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
586 ensure!(
587 info.get("rustc_fingerprint")
588 .is_some_and(serde_json::Value::is_u64)
589 && info
590 .get("outputs")
591 .and_then(serde_json::Value::as_object)
592 .is_some_and(|outputs| outputs.values().any(|output| {
593 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
594 && output
595 .get("stdout")
596 .and_then(serde_json::Value::as_str)
597 .is_some_and(|stdout| stdout.starts_with("rustc "))
598 && output
599 .get("stderr")
600 .is_some_and(serde_json::Value::is_string)
601 }))
602 && info
603 .get("successes")
604 .is_some_and(serde_json::Value::is_object),
605 "missing-tag rustc identity structure invalid: {}",
606 path.display()
607 );
608 let mut profiles = 0;
609 for profile in ["debug", "release"] {
610 let dir = target.join(profile);
611 if !dir.exists() {
612 continue;
613 }
614 profile_output_dirs(&dir, true)?;
615 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
616 profiles += 1;
617 }
618 ensure!(
619 profiles > 0,
620 "missing-tag proof: no Cargo profile in {}",
621 target.display()
622 );
623 }
624 for name in ["debug", "release"] {
625 let path = target.join(name);
626 match std::fs::symlink_metadata(&path) {
627 Ok(metadata) => ensure!(
628 metadata.is_dir() && path.canonicalize()? == path,
629 "profile validation: not a plain directory: {}",
630 path.display()
631 ),
632 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
633 Err(error) => {
634 return Err(error)
635 .with_context(|| format!("profile validation: inspect {}", path.display()));
636 }
637 }
638 }
639 Ok(target)
640}
641
642fn plain_file(path: &Path, stage: &str) -> Result<()> {
643 let meta = std::fs::symlink_metadata(path)
644 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
645 ensure!(
646 meta.is_file() && path.canonicalize()? == path,
647 "{stage}: not a plain file: {}",
648 path.display()
649 );
650 Ok(())
651}
652
653fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
654 let path = target.join("CACHEDIR.TAG");
655 match std::fs::symlink_metadata(&path) {
656 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
657 return Ok(());
658 }
659 result => {
660 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
661 }
662 }
663 plain_file(&path, "cache tag")?;
664 let tag = std::fs::read_to_string(&path)
665 .with_context(|| format!("cache tag: read {}", path.display()))?;
666 ensure!(
667 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
668 "unknown Cargo cache tag: {}",
669 path.display()
670 );
671 Ok(())
672}
673
674fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
678 let mut dirs = vec![target.to_owned()];
679 let mut entries = 0;
680 while let Some(dir) = dirs.pop() {
681 for entry in std::fs::read_dir(&dir)
682 .with_context(|| format!("output activity: list {}", dir.display()))?
683 {
684 let path = entry?.path();
685 entries += 1;
686 ensure!(
687 entries <= 100_000,
688 "output activity inventory too large: {}",
689 target.display()
690 );
691 let metadata = std::fs::symlink_metadata(&path)
692 .with_context(|| format!("output activity: inspect {}", path.display()))?;
693 if metadata.is_dir() {
694 if dir != target
696 || matches!(
697 path.file_name().and_then(|s| s.to_str()),
698 Some("debug" | "release")
699 )
700 {
701 dirs.push(path);
702 }
703 } else if metadata.is_file()
704 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
705 {
706 let at =
707 Timestamp::try_from(metadata.modified().with_context(|| {
708 format!("output activity: modified {}", path.display())
709 })?)?;
710 ensure!(
711 crate::clean::due(now, at, grace),
712 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
713 path.display()
714 );
715 }
716 }
717 }
718 Ok(())
719}
720
721pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
722 use std::io::Read as _;
723 let mut header = [0; 8];
724 let n = File::open(path)?.read(&mut header)?;
725 let header = &header[..n];
726 Ok(header.starts_with(b"\x7fELF")
727 || header.starts_with(b"!<arch>\n")
728 || header.starts_with(b"rust\0\0\0")
729 || header.starts_with(b"MZ\x90\0")
730 || [
731 [0xfe, 0xed, 0xfa, 0xce],
732 [0xce, 0xfa, 0xed, 0xfe],
733 [0xfe, 0xed, 0xfa, 0xcf],
734 [0xcf, 0xfa, 0xed, 0xfe],
735 [0xca, 0xfe, 0xba, 0xbe],
736 [0xca, 0xfe, 0xba, 0xbf],
737 ]
738 .iter()
739 .any(|magic| header.starts_with(magic)))
740}
741
742fn profile_output_dirs(profile: &Path, require_proof: bool) -> Result<Vec<PathBuf>> {
746 let mut present = std::collections::BTreeSet::new();
747 for path in [
748 profile.to_owned(),
749 profile.join("deps"),
750 profile.join(".fingerprint"),
751 profile.join("build"),
752 ] {
753 match std::fs::symlink_metadata(&path) {
754 Ok(_) => {
755 ensure!(
756 plain_dir(&path) && path.canonicalize()? == path,
757 "Cargo profile proof: {} is not a plain directory",
758 path.display()
759 );
760 present.insert(path);
761 }
762 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
763 Err(e) => return Err(e.into()),
764 }
765 }
766 if !present.contains(profile) {
767 return Ok(Vec::new());
768 }
769 let deps = present.contains(&profile.join("deps"));
770 let fingerprint = present.contains(&profile.join(".fingerprint"));
771 let mut dirs = vec![profile.to_owned()];
772 if deps || fingerprint {
773 ensure!(
774 !require_proof || (deps && fingerprint),
775 "incomplete Cargo v1 profile proof"
776 );
777 if deps {
778 dirs.push(profile.join("deps"));
779 }
780 if present.contains(&profile.join("build")) {
783 for entry in std::fs::read_dir(profile.join("build"))? {
784 let path = entry?.path();
785 if plain_dir(&path) {
786 for entry in std::fs::read_dir(&path)? {
787 ensure!(
788 !entry?.file_name().to_str().is_some_and(unit_hash),
789 "mixed Cargo profile layouts"
790 );
791 }
792 }
793 }
794 }
795 } else if present.contains(&profile.join("build")) {
796 let mut units = 0;
797 for package in std::fs::read_dir(profile.join("build"))? {
798 let package = package?.path();
799 let meta = std::fs::symlink_metadata(&package)?;
800 ensure!(
801 !meta.file_type().is_symlink(),
802 "Cargo v2 package is a symlink: {}",
803 package.display()
804 );
805 if !meta.is_dir() {
806 ensure!(!require_proof, "Cargo v2 package is not a directory");
807 continue;
808 }
809 ensure!(
810 package.canonicalize()? == package,
811 "Cargo v2 package is not a plain directory"
812 );
813 let mut package_units = 0;
814 for unit in std::fs::read_dir(&package)? {
815 let unit = unit?.path();
816 let meta = std::fs::symlink_metadata(&unit)?;
817 ensure!(
818 !meta.file_type().is_symlink(),
819 "Cargo v2 unit is a symlink: {}",
820 unit.display()
821 );
822 let structured = meta.is_dir()
826 && unit
827 .file_name()
828 .and_then(|n| n.to_str())
829 .is_some_and(unit_hash)
830 && unit.canonicalize()? == unit
831 && ["fingerprint", "out"].iter().all(|name| {
832 let path = unit.join(name);
833 plain_dir(&path) && path.canonicalize().is_ok_and(|p| p == path)
834 });
835 if structured && v2_run_script_unit(&unit) {
838 continue;
839 }
840 let complete = structured
841 && plain_file(
842 &unit.join("fingerprint/invoked.timestamp"),
843 "Cargo unit fingerprint",
844 )
845 .is_ok();
846 if !complete {
847 ensure!(!require_proof, "invalid or incomplete Cargo v2 unit");
848 continue;
849 }
850 dirs.push(unit.join("out"));
851 units += 1;
852 package_units += 1;
853 ensure!(units <= 4096, "Cargo unit inventory too large");
854 }
855 ensure!(
856 package_units > 0 || !require_proof,
857 "empty Cargo v2 package"
858 );
859 }
860 ensure!(units > 0 || !require_proof, "missing Cargo v2 units");
861 } else {
862 ensure!(!require_proof, "missing Cargo profile proof");
863 }
864 Ok(dirs)
865}
866
867fn v2_run_script_unit(unit: &Path) -> bool {
870 let fp = unit.join("fingerprint");
871 if std::fs::symlink_metadata(fp.join("invoked.timestamp")).is_ok() {
872 return false;
873 }
874 let Ok(entries) = std::fs::read_dir(&fp) else {
875 return false;
876 };
877 for entry in entries.flatten() {
878 let path = entry.path();
879 let name = path.file_name().unwrap_or_default().to_string_lossy();
880 if name.starts_with("run-build-script-")
881 && path.extension().is_none()
882 && plain_file(&path, "Cargo run fingerprint").is_ok()
883 && plain_file(&path.with_extension("json"), "Cargo run fingerprint json").is_ok()
884 {
885 return true;
886 }
887 }
888 false
889}
890
891fn v2_compiler_dep_info(dir: &Path, dep_info: &Path) -> Result<bool> {
894 let unit = dir.parent().context("Cargo unit parent")?;
895 let hash = unit.file_name().unwrap_or_default().to_string_lossy();
896 let stem = dep_info.file_stem().unwrap_or_default().to_string_lossy();
897 for entry in std::fs::read_dir(unit.join("fingerprint"))? {
898 let path = entry?.path();
899 let name = path.file_name().unwrap_or_default().to_string_lossy();
900 let library = name
903 .strip_prefix("lib-")
904 .is_some_and(|name| stem == format!("{}-{hash}", name.replace('-', "_")));
905 let binary = name
906 .strip_prefix("bin-")
907 .is_some_and(|name| stem == name.replace('-', "_"));
908 if library || binary {
909 plain_file(&path, "Cargo compiler fingerprint")?;
910 plain_file(
911 &path.with_extension("json"),
912 "Cargo compiler fingerprint json",
913 )?;
914 plain_file(
915 &path.with_file_name(format!(
916 "dep-{}",
917 path.file_name().unwrap().to_string_lossy()
918 )),
919 "Cargo compiler fingerprint dep-info",
920 )?;
921 return Ok(true);
922 }
923 }
924 Ok(false)
925}
926
927fn unit_hash(name: &str) -> bool {
928 name.len() == 16 && name.bytes().all(|b| b.is_ascii_hexdigit())
929}
930
931fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
935 let mut outputs = std::collections::BTreeSet::new();
936 for profile in ["debug", "release"] {
937 for dir in profile_output_dirs(&target.join(profile), false)? {
938 if !plain_dir(&dir) || dir.canonicalize()? != dir {
939 continue;
940 }
941 for entry in std::fs::read_dir(&dir)? {
942 let path = entry?.path();
943 if path.extension().is_none_or(|ext| ext != "d")
944 || !std::fs::symlink_metadata(&path)?.is_file()
945 || std::fs::metadata(&path)?.len() > 1024 * 1024
946 || path.canonicalize()? != path
947 {
948 continue;
949 }
950 if dir.file_name().is_some_and(|name| name == "out")
951 && !v2_compiler_dep_info(&dir, &path)?
952 {
953 continue;
954 }
955 let body = std::fs::read_to_string(&path)?;
956 for line in body.lines() {
957 let Some((names, dependencies)) = line.split_once(": ") else {
958 continue;
959 };
960 if dependencies.trim().is_empty() {
961 continue;
962 }
963 let mut words = Vec::new();
966 let mut word = String::new();
967 let mut escaped = false;
968 for c in names.chars() {
969 if escaped {
970 word.push(c);
971 escaped = false;
972 } else if c == '\\' {
973 escaped = true;
974 } else if c.is_whitespace() {
975 if !word.is_empty() {
976 words.push(std::mem::take(&mut word));
977 }
978 } else {
979 word.push(c);
980 }
981 }
982 if escaped || word.contains(['$', '%']) {
983 continue;
984 }
985 if !word.is_empty() {
986 words.push(word);
987 }
988 for name in words {
989 let output = PathBuf::from(name);
990 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
991 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
992 let artifact_name =
993 output_stem == stem || output_stem == format!("lib{stem}");
994 let artifact_kind = output.extension().is_none_or(|ext| {
995 matches!(
996 ext.to_str(),
997 Some(
998 "rlib"
999 | "rmeta"
1000 | "so"
1001 | "dylib"
1002 | "a"
1003 | "lib"
1004 | "dll"
1005 | "exe"
1006 | "pdb"
1007 )
1008 )
1009 });
1010 if artifact_name
1011 && artifact_kind
1012 && output.parent() == Some(dir.as_path())
1013 && output.extension().is_none_or(|ext| ext != "d")
1014 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
1015 && output.canonicalize().is_ok_and(|p| p == output)
1016 && artifact_signature(&output)?
1017 {
1018 outputs.insert(output);
1019 }
1020 }
1021 }
1022 }
1023 }
1024 }
1025 Ok(outputs.into_iter().collect())
1026}
1027
1028#[derive(Serialize, Deserialize)]
1029struct CargoResidue {
1030 target: PathBuf,
1031 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
1032}
1033
1034fn residue_files(
1037 target: &Path,
1038 outputs: &[PathBuf],
1039) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
1040 let mut files = std::collections::BTreeMap::new();
1041 let mut dirs = vec![target.to_owned()];
1042 let mut bytes = 0usize;
1043 let mut entries = 0usize;
1044 while let Some(dir) = dirs.pop() {
1045 for entry in std::fs::read_dir(dir)? {
1046 let path = entry?.path();
1047 entries += 1;
1048 ensure!(entries <= 4096, "residue inventory too large");
1049 let metadata = std::fs::symlink_metadata(&path)?;
1050 ensure!(
1051 !metadata.file_type().is_symlink(),
1052 "symlink in Cargo residue"
1053 );
1054 if metadata.is_dir() {
1055 dirs.push(path);
1056 } else {
1057 ensure!(metadata.is_file(), "non-file in Cargo residue");
1058 if outputs.contains(&path) {
1059 continue;
1060 }
1061 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
1062 let body = std::fs::read(&path)?;
1063 bytes = bytes.saturating_add(body.len());
1064 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
1065 files.insert(path.strip_prefix(target)?.to_owned(), body);
1066 }
1067 }
1068 }
1069 Ok(files)
1070}
1071
1072fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
1073 Ok(home
1074 .join("runs")
1075 .join(id)
1076 .join("build-residue")
1077 .join(worktree.file_name().context("missing worktree name")?))
1078}
1079
1080fn residue_root(target: &Path) -> Result<()> {
1083 for entry in std::fs::read_dir(target)? {
1084 let name = entry?.file_name();
1085 ensure!(
1086 matches!(
1087 name.to_str(),
1088 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
1089 ),
1090 "unrecognized Cargo residue root entry"
1091 );
1092 }
1093 Ok(())
1094}
1095
1096fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
1097 cache_tag(target, false)?;
1098 residue_root(target)?;
1099 let files = residue_files(target, outputs)?;
1100 for (relative, body) in &files {
1101 let path = target.join(relative);
1102 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
1103 let lock = matches!(
1104 relative.to_str(),
1105 Some("debug/.cargo-lock" | "release/.cargo-lock")
1106 ) && body.is_empty();
1107 let dep_info = path.extension().is_some_and(|ext| ext == "d")
1108 && outputs.iter().any(|output| {
1109 output.parent() == path.parent()
1110 && (output.file_stem() == path.file_stem()
1111 || output.file_stem().is_some_and(|stem| {
1112 stem.to_string_lossy()
1113 == format!(
1114 "lib{}",
1115 path.file_stem().unwrap_or_default().to_string_lossy()
1116 )
1117 }))
1118 });
1119 ensure!(identity || lock || dep_info, "unproven Cargo residue");
1120 }
1121 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
1122 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
1123 let tmp = proof.with_extension("tmp");
1124 std::fs::write(
1125 &tmp,
1126 serde_json::to_vec(&CargoResidue {
1127 target: target.to_owned(),
1128 files,
1129 })?,
1130 )?;
1131 std::fs::rename(tmp, proof)?;
1132 Ok(())
1133}
1134
1135pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
1136 let target = cargo_target(worktree).await?;
1137 residue_root(&target)?;
1138 let locks = cargo_locks(&target)?;
1139 let proof: CargoResidue =
1140 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
1141 ensure!(
1142 proof.target == target && proof.files == residue_files(&target, &[])?,
1143 "Cargo residue changed or contains unproven files"
1144 );
1145 Ok(locks)
1146}
1147
1148pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
1150 let mut held = Vec::new();
1151 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
1152 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
1153 }) {
1154 let lock = target.join(profile).join(name);
1155 match std::fs::symlink_metadata(&lock) {
1156 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
1157 result => {
1158 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
1159 }
1160 }
1161 {
1162 plain_file(&lock, "Cargo lock")?;
1163 let file = OpenOptions::new()
1164 .read(true)
1165 .write(true)
1166 .open(&lock)
1167 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
1168 crate::pr_head::try_lock_file(&file)
1169 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
1170 held.push(file);
1171 }
1172 }
1173 Ok(held)
1174}
1175
1176pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
1179 let home = home.to_owned();
1180 let disk = disk.clone();
1181 let runtime = tokio::runtime::Handle::current();
1182 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
1183 .await
1184 .unwrap_or_else(|e| Report {
1185 failures: vec![format!("retention worker failed: {e}")],
1186 ..Report::default()
1187 })
1188}
1189
1190#[derive(Default, Serialize, Deserialize)]
1191struct Measurement {
1192 at: Option<Timestamp>,
1193 bytes: u64,
1194}
1195
1196async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
1197 let mut report = Report {
1198 checked_at: Some(now),
1199 limit_bytes: disk.worktree_limit_bytes,
1200 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
1201 ..Report::default()
1202 };
1203 if !cfg!(unix) {
1204 report
1205 .failures
1206 .push("Automatic output reclamation requires Unix process containment".into());
1207 return report;
1208 }
1209 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
1210 return report;
1211 };
1212 let mut targets = Vec::new();
1213 let mut under_pressure = false;
1214 let mut registrations = std::collections::HashMap::new();
1215 let measurements_path = home.join("output-measurements.json");
1216 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
1217 std::fs::read(&measurements_path)
1218 .ok()
1219 .and_then(|b| serde_json::from_slice(&b).ok())
1220 .unwrap_or_default();
1221 let mut observed = std::collections::BTreeSet::new();
1222 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
1223 .into_iter()
1224 .flatten()
1225 .flatten()
1226 .map(|e| e.file_name().to_string_lossy().into_owned())
1227 .collect();
1228 ids.sort();
1229 for id in ids {
1230 if !crate::run::is_run_id(&id) {
1231 continue;
1232 }
1233 let state = match RunState::load_under(&id, home) {
1234 Ok(state) => state,
1235 Err(e) => {
1236 report
1237 .retained
1238 .push(format!("{id}: unreadable run ownership: {e:#}"));
1239 continue;
1240 }
1241 };
1242 if disk.worktree_limit_bytes == 0 {
1243 continue;
1244 }
1245 match run_paths_missing(&state) {
1246 Ok(true) => continue,
1247 Ok(false) => {}
1248 Err(e) => {
1249 report.failures.push(format!("{id}: {e:#}"));
1250 continue;
1251 }
1252 }
1253 let list = match registrations.entry(state.repo.clone()) {
1254 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1255 std::collections::hash_map::Entry::Vacant(entry) => {
1256 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1257 Ok(list) => entry.insert(list),
1258 Err(e) => {
1259 if matches!(run_paths_missing(&state), Ok(true)) {
1262 continue;
1263 }
1264 report.failures.push(format!("{id}: {e:#}"));
1265 continue;
1266 }
1267 }
1268 }
1269 };
1270 let paths = match registered_worktrees(&state, list) {
1271 Ok(paths) => paths,
1272 Err(e) => {
1273 report.failures.push(format!("{id}: {e:#}"));
1274 continue;
1275 }
1276 };
1277 for path in paths {
1278 if !path.join("target").exists() {
1279 continue;
1280 }
1281 let target = path.join("target");
1282 observed.insert(target.clone());
1283 let pressure =
1284 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1285 under_pressure |= pressure;
1286 let measurement = measurements.entry(target).or_default();
1287 if pressure
1288 || measurement
1289 .at
1290 .is_none_or(|at| crate::clean::due(now, at, 300))
1291 {
1292 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1293 measurement.at = Some(now);
1294 }
1295 let size = measurement.bytes;
1296 report.measured_bytes = report.measured_bytes.saturating_add(size);
1297 targets.push((state.output_activity(), id.clone(), path, size));
1298 }
1299 }
1300 if under_pressure {
1301 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1304 } else {
1305 targets.sort_by_key(|t| t.0);
1306 }
1307 let free_before = crate::disk::free_bytes(home).ok();
1308 #[cfg(unix)]
1309 {
1310 let mut seen = std::collections::BTreeSet::new();
1311 report.measured_allocated_file_bytes = Some(0);
1312 report.estimated_released_allocated_bytes = Some(0);
1313 report.proven_reclaimable_allocated_bytes = Some(0);
1314 for (_, _, path, _) in &targets {
1315 report.measured_allocated_file_bytes = report
1316 .measured_allocated_file_bytes
1317 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1318 .map(|(a, b)| a.saturating_add(b));
1319 }
1320 }
1321 let mut remaining = report.measured_bytes;
1322 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1323 let mut deletion_started = false;
1324 let mut oversized_started = false;
1325 for (updated, id, path, _) in targets {
1326 if disk.worktree_limit_bytes == 0
1327 || remaining <= disk.worktree_limit_bytes
1328 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1329 {
1330 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1331 continue;
1332 }
1333 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1334 report
1335 .retained
1336 .push(format!("{id}: per-pass byte budget spent"));
1337 continue;
1338 }
1339 let result = async {
1340 ensure!(
1341 crate::clean::due(now, updated, disk.fold_grace_secs),
1342 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1343 );
1344 let _owner = crate::pr_head::owner_lock(home, &id)?;
1345 let state = RunState::load_under(&id, home)?;
1346 ensure!(
1347 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1348 == Liveness::Dead,
1349 "run live or ownership unknown"
1350 );
1351 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1352 let _driver = crate::driver::maintenance(home, &id)?;
1353 let _outputs = OutputLease::take(&path, true)?;
1354 ensure!(
1355 worktrees(&state).await?.contains(&path),
1356 "worktree registration changed"
1357 );
1358 let target = validate_cargo_target(&path, true).await
1359 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1360 let _cargo = cargo_locks(&target)
1361 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1362 validate_cargo_target(&path, true).await
1365 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1366 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1367 if lease_meta.len() > 0 {
1368 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1369 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1370 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1371 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1372 }
1373 output_age(&target, now, disk.fold_grace_secs)
1374 .with_context(|| format!("stage output-age: {}", target.display()))?;
1375 let mut outputs = declared_outputs(&target)
1376 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1377 let incremental = match crate::incremental::candidates(&path) {
1378 Ok(candidates) => candidates,
1379 Err(error) => {
1380 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1381 crate::incremental::Reclaim::default()
1382 }
1383 };
1384 let incremental_files = incremental.files.clone();
1385 outputs.extend(incremental_files.iter().cloned());
1386 outputs.sort();
1387 outputs.dedup();
1388 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1389 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1390 .with_context(|| format!("stage measure: {}", p.display())))
1391 .collect::<Result<Vec<_>>>()?;
1392 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1393 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1394 let mut allocation = ReleasedAllocation::of(&outputs)?;
1395 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1396 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1397 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1398 ensure!(!deletion_started || !exceeds_budget,
1402 "declared outputs exceed remaining per-pass byte budget");
1403 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1406 std::fs::write(
1407 &ledger,
1408 serde_json::to_vec(
1409 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1410 )?,
1411 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1412 if !incremental_files.is_empty() {
1415 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1416 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1417 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1418 }
1419 ensure!(worktrees(&state).await?.contains(&path),
1420 "worktree registration changed before deletion");
1421 deletion_started = true;
1422 oversized_started = exceeds_budget;
1423 measurements.remove(&target);
1424 for (output, size) in outputs.iter().zip(sizes) {
1425 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1426 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1427 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1428 report.freed_bytes = report.freed_bytes.saturating_add(size);
1430 remaining = remaining.saturating_sub(size);
1431 }
1432 incremental.finish().context("finish incremental reclamation")?;
1433 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1434 Ok::<_, anyhow::Error>(())
1435 }
1436 .await;
1437 if let Err(e) = result {
1438 report.retained.push(format!(
1439 "{id}/{}: {e:#}",
1440 path.file_name().unwrap_or_default().to_string_lossy()
1441 ));
1442 }
1443 }
1444 measurements.retain(|path, _| observed.contains(path));
1445 if let Ok(body) = serde_json::to_vec(&measurements)
1446 && let Err(e) = std::fs::write(&measurements_path, body)
1447 {
1448 report.failures.push(format!("output measurements: {e}"));
1449 }
1450 copies::evict(home, disk, now, &mut report).await;
1451 retire_empty(home, disk, now, &mut report).await;
1452 report.protected_or_unproven_bytes = report
1453 .measured_bytes
1454 .saturating_sub(report.proven_reclaimable_bytes);
1455 let pressure_remaining = pressure_paths
1456 .iter()
1457 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1458 if !report.failures.is_empty()
1459 || disk.worktree_limit_bytes > 0
1460 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1461 {
1462 let details = report
1463 .retained
1464 .iter()
1465 .filter(|reason| {
1466 !reason.contains("within policy")
1467 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1468 })
1469 .take(4)
1470 .chain(report.failures.iter().take(2))
1471 .cloned()
1472 .collect::<Vec<_>>()
1473 .join("; ");
1474 crate::notices::raise_in(
1475 home,
1476 crate::notices::Notice::warn(
1477 "disk:retention",
1478 format!(
1479 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1480 report.measured_bytes,
1481 report.proven_reclaimable_bytes,
1482 report.protected_or_unproven_bytes,
1483 report.freed_bytes,
1484 if report.proven_reclaimable_bytes == 0 {
1485 "No inspected contents have deletion authority."
1486 } else {
1487 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1488 }
1489 ),
1490 ),
1491 );
1492 }
1493 report.observed_home_free_change_bytes = free_before
1494 .zip(crate::disk::free_bytes(home).ok())
1495 .map(|(before, after)| {
1496 (i128::from(after) - i128::from(before))
1497 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1498 });
1499 let store = home.join("retention.json");
1500 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1501 let tmp = home.join("retention.json.tmp");
1502 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1503 tracing::warn!("retention report could not be saved: {e}");
1504 }
1505 }
1506 tracing::info!(
1507 measured_bytes = report.measured_bytes,
1508 freed_bytes = report.freed_bytes,
1509 retired = report.retired,
1510 retained = report.retained.len(),
1511 failures = report.failures.len(),
1512 "retained worktree cleanup"
1513 );
1514 for reason in report.retained.iter().chain(&report.failures) {
1515 tracing::debug!(%reason, "worktree retained");
1516 }
1517 report
1518}
1519
1520async fn retire_empty(
1521 home: &Path,
1522 disk: &crate::config::Disk,
1523 now: Timestamp,
1524 report: &mut Report,
1525) {
1526 if !disk.auto_fold {
1527 return;
1528 }
1529 let queue = crate::queue::Queue::at(home.join("queue"));
1530 let questions = crate::ask::Questions::at(home.join("questions"));
1531 let open = questions.list_for_effects();
1532 for listed in queue.list() {
1533 let Ok(_claim) = queue.claim(&listed.id) else {
1534 continue;
1535 };
1536 let Ok(task) = queue.get(&listed.id) else {
1537 continue;
1538 };
1539 if open
1540 .iter()
1541 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1542 {
1543 report.retained.push(format!(
1544 "{}: task has an unanswered owner decision",
1545 task.id
1546 ));
1547 continue;
1548 }
1549 let sources: Vec<_> = task
1550 .runs
1551 .iter()
1552 .collect::<std::collections::BTreeSet<_>>()
1553 .into_iter()
1554 .cloned()
1555 .collect();
1556 let mut useful = Vec::new();
1557 let mut selected = Vec::new();
1558 let mut public = Vec::new();
1559 let mut unreadable = false;
1560 for id in &sources {
1561 let Ok(state) = RunState::load_under(id, home) else {
1562 unreadable = true;
1563 continue;
1564 };
1565 if state.released() || state.retired_to.is_some() {
1566 continue;
1567 }
1568 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1569 selected.push(state.clone());
1570 }
1571 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1572 public.push(state.clone());
1573 }
1574 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1575 useful.push(state);
1576 }
1577 }
1578 let pool = if !selected.is_empty() {
1582 selected
1583 } else if !public.is_empty() {
1584 public
1585 } else {
1586 useful
1587 };
1588 if unreadable || pool.len() != 1 {
1589 report.retained.push(format!(
1590 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1591 task.id,
1592 sources.len()
1593 ));
1594 continue;
1595 }
1596 let authority = &pool[0];
1597 if authority.candidates.is_empty() {
1598 continue;
1599 }
1600 let latest = &authority.id;
1601 for id in &sources {
1602 if id == latest || report.retired >= 5 {
1603 continue;
1604 }
1605 if let Ok(state) = RunState::load_under(id, home)
1606 && state.retired_to.as_ref() == Some(latest)
1607 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1608 {
1609 continue;
1610 }
1611 let result = async {
1612 ensure!(
1613 !task
1614 .resume_override
1615 .as_ref()
1616 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1617 "operator-pinned recovery run"
1618 );
1619 ensure!(
1620 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1621 "selected replacement recovery source"
1622 );
1623 let _owner = crate::pr_head::owner_lock(home, id)?;
1624 let mut state = RunState::load_under(id, home)?;
1625 ensure!(
1626 state.repo == authority.repo,
1627 "repository differs from recovery authority"
1628 );
1629 ensure!(
1630 !crate::pr_recovery::explicit_sources_retained(home, id),
1631 "explicitly retained recovery source"
1632 );
1633 ensure!(
1634 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1635 == Liveness::Dead,
1636 "run live or ownership unknown"
1637 );
1638 let _driver = crate::driver::maintenance(home, id)?;
1639 ensure!(
1640 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1641 "within grace period"
1642 );
1643 ensure!(
1644 !open.iter().any(|q| q.run == *id && q.status.open()),
1645 "unanswered owner decision"
1646 );
1647 ensure!(
1648 state.pr.is_none()
1649 && state.pr_head.is_none()
1650 && state.pr_recovery.is_none()
1651 && state.land_approval.is_none()
1652 && state.merge_decision.is_none()
1653 && state.reviews.is_empty()
1654 && state.recovery.is_none()
1655 && state.candidate_preservation.is_empty()
1656 && state.interrupted_implementations.is_empty()
1657 && !state.released(),
1658 "PR ownership or recovery checkpoint"
1659 );
1660 let proof = prove_empty(home, &state, latest).await?;
1661 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1662 for path in &proof.paths {
1665 ensure!(
1666 git::worktree_remove_clean(&state.repo, path).await?,
1667 "Git refused clean worktree removal"
1668 );
1669 }
1670 Ok::<_, anyhow::Error>(())
1671 }
1672 .await;
1673 match result {
1674 Ok(()) => report.retired += 1,
1675 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1676 }
1677 }
1678 }
1679}
1680
1681struct EmptyProof {
1682 paths: Vec<PathBuf>,
1683 _outputs: Vec<OutputLease>,
1684 _cargo: Vec<File>,
1685}
1686
1687pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1690 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1691 let mut files = Vec::new();
1692 let mut blobs = Vec::new();
1693 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1694 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1695 let parts: Vec<_> = header.split_whitespace().collect();
1696 ensure!(
1697 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1698 "unsupported tracked source kind"
1699 );
1700 let path = worktree.join(name);
1701 let meta = std::fs::symlink_metadata(&path)?;
1702 ensure!(
1703 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1704 "noncanonical tracked source"
1705 );
1706 #[cfg(unix)]
1707 {
1708 use std::os::unix::fs::PermissionsExt as _;
1709 ensure!(
1710 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1711 "tracked executable mode differs from original base"
1712 );
1713 }
1714 path.to_str().context("non UTF-8 source")?;
1715 files.push(git::path_arg(&path));
1716 blobs.push(parts[2].to_owned());
1717 }
1718 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1719 let mut args = vec!["hash-object", "--no-filters", "--"];
1720 args.extend(paths.iter().map(String::as_str));
1721 let actual = git::git(worktree, &args).await?;
1722 ensure!(
1723 actual.lines().eq(expected.iter().map(String::as_str)),
1724 "tracked bytes differ from original base"
1725 );
1726 }
1727 Ok(())
1728}
1729
1730async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1731 let id = &state.id;
1732 let paths = worktrees(state).await?;
1733 let mut guards = Vec::new();
1734 let mut cargo_guards = Vec::new();
1735 for path in &paths {
1736 guards.push(OutputLease::take(path, true)?);
1737 ensure!(
1738 git::rev_parse(path, "HEAD").await? == state.base_commit,
1739 "unique or unrecorded commits"
1740 );
1741 ensure!(
1742 git::git(
1743 path,
1744 &[
1745 "--no-optional-locks",
1746 "status",
1747 "--porcelain",
1748 "--untracked-files=all"
1749 ]
1750 )
1751 .await?
1752 .is_empty(),
1753 "tracked or untracked source changes"
1754 );
1755 ensure!(
1756 git::git(
1757 path,
1758 &[
1759 "--no-optional-locks",
1760 "status",
1761 "--porcelain",
1762 "--untracked-files=all",
1763 "--ignored",
1764 "--",
1765 ".",
1766 ":(exclude)target"
1767 ]
1768 )
1769 .await?
1770 .is_empty(),
1771 "source changes, ignored files or build outputs"
1772 );
1773 tracked_base(path, &state.base_commit).await?;
1774 match std::fs::symlink_metadata(path.join("target")) {
1775 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1776 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1777 Err(error) => return Err(error.into()),
1778 }
1779 for checkpoint in [
1780 "rebase-merge",
1781 "rebase-apply",
1782 "sequencer",
1783 "BISECT_LOG",
1784 "REVERT_HEAD",
1785 "AUTO_MERGE",
1786 "MERGE_MSG",
1787 "CHERRY_PICK_HEAD",
1788 "MERGE_HEAD",
1789 "REBASE_HEAD",
1790 ] {
1791 let location = git::git(
1792 path,
1793 &[
1794 "rev-parse",
1795 "--path-format=absolute",
1796 "--git-path",
1797 checkpoint,
1798 ],
1799 )
1800 .await?;
1801 ensure!(
1802 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1803 "retained Git recovery metadata: {checkpoint}"
1804 );
1805 }
1806 let origin_path = git::git(
1809 path,
1810 &[
1811 "rev-parse",
1812 "--path-format=absolute",
1813 "--git-path",
1814 "ORIG_HEAD",
1815 ],
1816 )
1817 .await?;
1818 match std::fs::symlink_metadata(origin_path.trim()) {
1819 Ok(meta) => {
1820 ensure!(
1821 meta.is_file() && !meta.file_type().is_symlink(),
1822 "unknown ORIG_HEAD artifact"
1823 );
1824 git::rev_parse(path, "ORIG_HEAD")
1825 .await
1826 .context("unreadable ORIG_HEAD")?;
1827 }
1828 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1829 Err(e) => return Err(e.into()),
1830 }
1831 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1832 ensure!(
1833 git::git_raw(
1834 path,
1835 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1836 )
1837 .await?
1838 .ok(),
1839 "ORIG_HEAD retains commits outside the original base"
1840 );
1841 }
1842 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1843 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1844 ensure!(
1845 git::git_raw(
1846 path,
1847 &["merge-base", "--is-ancestor", head, &state.base_commit]
1848 )
1849 .await?
1850 .ok(),
1851 "worktree reflog retains commits outside the original base"
1852 );
1853 }
1854 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1855 ensure!(
1856 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1857 .await?
1858 .ok(),
1859 "unfinished Git operation"
1860 );
1861 }
1862 }
1863 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1866 return Ok(EmptyProof {
1867 paths,
1868 _outputs: guards,
1869 _cargo: cargo_guards,
1870 });
1871 }
1872 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1873 for candidate in &state.candidates {
1874 ensure!(
1875 (paths.contains(&candidate.worktree)
1876 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1877 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1878 "candidate recovery source"
1879 );
1880 }
1881 let pattern = format!("refs/heads/magi/{}/", state.short());
1882 let branches = git::git(
1883 &state.repo,
1884 &["for-each-ref", "--format=%(objectname)", &pattern],
1885 )
1886 .await?;
1887 ensure!(
1888 branches.lines().all(|head| head == state.base_commit),
1889 "unrecorded candidate branch changes"
1890 );
1891 Ok(EmptyProof {
1892 paths,
1893 _outputs: guards,
1894 _cargo: cargo_guards,
1895 })
1896}
1897
1898fn commit_retirement(
1903 home: &Path,
1904 queue: &crate::queue::Queue,
1905 expected: &crate::queue::Task,
1906 state: &mut RunState,
1907 authority: &str,
1908 sources: &[String],
1909 now: Timestamp,
1910) -> Result<()> {
1911 let _write = queue.lock_task(&expected.id)?;
1912 let current = queue.get(&expected.id)?;
1913 ensure!(
1914 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1915 "task changed during retirement proof; retaining worktrees for the current owner decision"
1916 );
1917 ensure!(
1918 !current
1919 .resume_override
1920 .as_ref()
1921 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1922 && !current
1923 .replacement
1924 .as_ref()
1925 .is_some_and(|pin| pin.run == state.id),
1926 "current owner selected this recovery source"
1927 );
1928 ensure!(
1929 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1930 "explicitly retained recovery source"
1931 );
1932 ensure!(
1933 !crate::ask::Questions::at(home.join("questions"))
1934 .list_for_effects()
1935 .iter()
1936 .any(|q| q.status.open()
1937 && (q.run == state.id
1938 || q.run == current.id
1939 || current.blocked_by.contains(&q.id))),
1940 "owner decision arrived during retirement proof"
1941 );
1942 let dir = home.join("recovery-lineages");
1943 std::fs::create_dir_all(&dir)?;
1944 let tmp = dir.join(format!("{}.tmp", current.id));
1945 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1946 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1947 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1948 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1949 state.retired_to = Some(authority.to_owned());
1950 state.event(
1951 "retention",
1952 format!(
1953 "empty attempt retired to {authority}; recovery sources: {}",
1954 sources.join(", ")
1955 ),
1956 );
1957 state.save_under(home)?;
1958 Ok(())
1960}
1961
1962pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1966 if min_free == 0 {
1967 return std::future::pending().await;
1968 }
1969 loop {
1970 for path in &paths {
1971 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1972 match crate::disk::free_bytes(existing) {
1973 Ok(free) if free >= min_free / 2 => {}
1974 Ok(free) => {
1975 return format!(
1976 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1977 );
1978 }
1979 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1980 }
1981 }
1982 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1983 }
1984}
1985
1986#[cfg(all(test, unix))]
1987mod tests {
1988 use super::*;
1989 use crate::{
1990 config::{Config, Disk},
1991 queue::{Source, Task},
1992 run::{Candidate, RunStatus},
1993 };
1994
1995 struct Fx {
1996 _temp: tempfile::TempDir,
1997 home: PathBuf,
1998 repo: PathBuf,
1999 bay: PathBuf,
2000 base: String,
2001 }
2002 impl Fx {
2003 async fn new() -> Self {
2004 let temp = tempfile::tempdir().unwrap();
2005 let root = temp.path().canonicalize().unwrap();
2006 let home = root.join("home");
2007 let repo = root.join("repo");
2008 let bay = root.join("worktrees");
2009 std::fs::create_dir_all(&repo).unwrap();
2010 std::fs::create_dir_all(home.join("runs")).unwrap();
2011 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
2012 git::git(&repo, &["config", "user.name", "Fixture"])
2013 .await
2014 .unwrap();
2015 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
2016 .await
2017 .unwrap();
2018 git::git(&repo, &["config", "commit.gpgsign", "false"])
2019 .await
2020 .unwrap();
2021 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
2022 std::fs::write(repo.join("source"), "base\n").unwrap();
2023 git::git(&repo, &["add", "."]).await.unwrap();
2024 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
2025 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
2026 Self {
2027 _temp: temp,
2028 home,
2029 repo,
2030 bay,
2031 base,
2032 }
2033 }
2034 async fn run(&self, tag: &str, useful: bool) -> RunState {
2035 let mut cfg = Config::default();
2036 cfg.graph.worktree_root = Some(self.bay.clone());
2037 let mut state = RunState::new(
2038 self.repo.clone(),
2039 "main".into(),
2040 self.base.clone(),
2041 "task".into(),
2042 cfg,
2043 );
2044 state.id = format!("20260101-000000-{tag}");
2045 state.status = RunStatus::Blocked;
2046 state.driver_exited = true;
2047 let wt = state.worktree_root().join("cand-A");
2048 let branch = format!("magi/{tag}/A");
2049 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
2050 .await
2051 .unwrap();
2052 if useful {
2053 std::fs::write(wt.join("source"), tag).unwrap();
2054 git::git(&wt, &["add", "."]).await.unwrap();
2055 git::git(&wt, &["commit", "-m", "candidate change"])
2056 .await
2057 .unwrap();
2058 }
2059 state.candidates.push(Candidate {
2060 index: 0,
2061 label: 'A',
2062 agent: "fixture".into(),
2063 branch,
2064 worktree: wt,
2065 summary: String::new(),
2066 stat: String::new(),
2067 files: usize::from(useful),
2068 commits: usize::from(useful),
2069 empty: !useful,
2070 failed: None,
2071 verified_noop: None,
2072 duration_ms: 0,
2073 folded: false,
2074 });
2075 state.save_under(&self.home).unwrap();
2076 state
2077 }
2078 fn disk() -> Disk {
2079 Disk {
2080 worktree_limit_bytes: 1,
2081 worktree_reclaim_bytes: 1024 * 1024,
2082 fold_grace_secs: 0,
2083 min_free_bytes: 0,
2084 ..Disk::default()
2085 }
2086 }
2087 fn target(wt: &Path) {
2088 let target = wt.join("target");
2089 std::fs::create_dir_all(target.join("debug")).unwrap();
2090 let mut artifact = [1; 200];
2091 artifact[..4].copy_from_slice(b"\x7fELF");
2092 std::fs::write(target.join("debug/output"), artifact).unwrap();
2093 let output = target
2094 .join("debug/output")
2095 .display()
2096 .to_string()
2097 .replace('\\', "\\\\")
2098 .replace(' ', "\\ ");
2099 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
2100 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
2101 std::fs::write(
2102 target.join("CACHEDIR.TAG"),
2103 "Signature: 8a477f597d28d172789f06886806bc55\n",
2104 )
2105 .unwrap();
2106 std::fs::write(
2107 target.join(".rustc_info.json"),
2108 "{\"rustc_fingerprint\":123}",
2109 )
2110 .unwrap();
2111 }
2112 fn now() -> Timestamp {
2113 Timestamp::now() + jiff::SignedDuration::from_secs(2)
2114 }
2115 fn task(&self, runs: &[RunState]) -> Task {
2116 let mut task = Task::new(
2117 "fixture".into(),
2118 "task".into(),
2119 self.repo.clone(),
2120 Source::Human,
2121 );
2122 task.runs = runs.iter().map(|s| s.id.clone()).collect();
2123 crate::queue::Queue::at(self.home.join("queue"))
2124 .put(&mut task)
2125 .unwrap();
2126 task
2127 }
2128 }
2129
2130 #[cfg(unix)]
2131 #[tokio::test]
2132 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
2133 let f = Fx::new().await;
2134 let state = f.run("ca01", true).await;
2135 let wt = &state.candidates[0].worktree;
2136 let generation = crate::incremental::tests::generate(wt);
2137 assert!(!declared_outputs(&wt.join("target")).unwrap().is_empty());
2139 for dir in profile_output_dirs(&wt.join("target/debug"), true).unwrap() {
2140 for entry in std::fs::read_dir(dir).unwrap() {
2141 let path = entry.unwrap().path();
2142 if path.extension().is_some_and(|x| x == "d") {
2143 std::fs::remove_file(path).unwrap();
2144 }
2145 }
2146 }
2147 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
2148 assert_eq!(
2149 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
2150 0,
2151 "live generation is protected"
2152 );
2153 drop(generation);
2154 let candidates = crate::incremental::candidates(wt).unwrap();
2155 let logical: u64 = candidates
2156 .files
2157 .iter()
2158 .map(|p| std::fs::metadata(p).unwrap().len())
2159 .sum();
2160 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
2161 std::fs::write(&unknown, "retain unknown data").unwrap();
2162 drop(candidates);
2163 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2164 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
2165 assert!(
2166 refused
2167 .retained
2168 .iter()
2169 .any(|s| s.contains("provenance mismatch"))
2170 );
2171 std::fs::remove_file(&unknown).unwrap();
2172 let legacy = wt.join("target/debug/incremental/legacy-session");
2173 std::fs::create_dir(&legacy).unwrap();
2174 let candidates = crate::incremental::candidates(wt).unwrap();
2176 let bin = candidates
2177 .files
2178 .iter()
2179 .find(|p| p.extension().is_some_and(|x| x == "bin"))
2180 .unwrap();
2181 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
2182 drop(candidates);
2183 let cargo = cargo_locks(&wt.join("target")).unwrap();
2184 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2185 drop(cargo);
2186 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
2187 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2188 drop(outputs);
2189 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2190 assert_eq!(report.freed_bytes, logical, "{report:?}");
2191 assert_eq!(report.proven_reclaimable_bytes, logical);
2192 assert!(report.measured_bytes > logical);
2193 assert_eq!(
2194 report.protected_or_unproven_bytes,
2195 report.measured_bytes - logical
2196 );
2197 #[cfg(unix)]
2198 {
2199 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
2200 assert!(
2201 report.measured_allocated_file_bytes.unwrap()
2202 >= report.estimated_released_allocated_bytes.unwrap()
2203 );
2204 }
2205 assert!(report.observed_home_free_change_bytes.is_some());
2206 assert!(legacy.join("query-cache.bin").is_file());
2207 assert!(wt.join("src/lib.rs").is_file());
2208 assert_eq!(report.retired, 0);
2209 }
2210
2211 #[cfg(unix)]
2212 #[tokio::test]
2213 async fn incremental_and_deps_links_account_for_retained_third_link() {
2214 use std::os::unix::fs::MetadataExt;
2215 let f = Fx::new().await;
2216 let state = f.run("ca02", true).await;
2217 let wt = &state.candidates[0].worktree;
2218 let generation = crate::incremental::tests::generate(wt);
2219 drop(generation);
2220 let candidates = crate::incremental::candidates(wt).unwrap();
2221 let objects = candidates
2222 .files
2223 .iter()
2224 .filter(|p| p.extension().is_some_and(|x| x == "o"))
2225 .take(2)
2226 .cloned()
2227 .collect::<Vec<_>>();
2228 assert_eq!(
2229 objects.len(),
2230 2,
2231 "fixture must produce real incremental objects"
2232 );
2233 let mut outputs = candidates.files.clone();
2234 drop(candidates);
2235 let target = wt.join("target");
2236 assert!(!declared_outputs(&target).unwrap().is_empty());
2237 let generated_dirs = profile_output_dirs(&target.join("debug"), true).unwrap();
2238 let mut output_dir = None;
2241 for dir in &generated_dirs {
2242 for entry in std::fs::read_dir(dir).unwrap() {
2243 let path = entry.unwrap().path();
2244 if path.extension().is_some_and(|x| x == "d") {
2245 std::fs::remove_file(path).unwrap();
2246 if dir != &generated_dirs[0] {
2247 output_dir = Some(dir.clone());
2248 }
2249 }
2250 }
2251 }
2252 let output_dir = output_dir.expect("Cargo generated no recognised output directory");
2253 for entry in std::fs::read_dir(&output_dir).unwrap() {
2257 let path = entry.unwrap().path();
2258 if path.is_file() {
2259 std::fs::remove_file(path).unwrap();
2260 }
2261 }
2262 let stem = if output_dir.file_name().is_some_and(|n| n == "out") {
2265 let unit = output_dir.parent().unwrap();
2266 let hash = unit.file_name().unwrap().to_string_lossy().into_owned();
2267 let name = std::fs::read_dir(unit.join("fingerprint"))
2268 .unwrap()
2269 .filter_map(|e| e.unwrap().file_name().into_string().ok())
2270 .find_map(|n| {
2271 n.strip_prefix("lib-")
2272 .filter(|n| !n.contains('.'))
2273 .map(str::to_owned)
2274 })
2275 .expect("Cargo unit has a lib fingerprint");
2276 format!("{}-{hash}", name.replace('-', "_"))
2277 } else {
2278 "link".to_owned()
2279 };
2280 let mut declared = Vec::new();
2283 for (index, object) in objects.iter().enumerate() {
2284 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2285 let dep = output_dir.join(if index == 0 {
2286 stem.clone()
2287 } else {
2288 format!("lib{stem}")
2289 });
2290 std::fs::hard_link(object, &dep).unwrap();
2291 declared.push(dep.display().to_string());
2292 outputs.push(dep);
2293 }
2294 std::fs::write(
2295 output_dir.join(format!("{stem}.d")),
2296 format!("{}: src/lib.rs\n", declared.join(" ")),
2297 )
2298 .unwrap();
2299 let retained = wt.join("retained-input");
2300 std::fs::hard_link(&objects[1], &retained).unwrap();
2301 let sentinel = std::fs::read(&retained).unwrap();
2302 let logical: u64 = outputs
2303 .iter()
2304 .map(|p| std::fs::metadata(p).unwrap().len())
2305 .sum();
2306 let mut seen = std::collections::BTreeSet::new();
2307 let retained_meta = std::fs::metadata(&retained).unwrap();
2308 let retained_inode = (retained_meta.dev(), retained_meta.ino());
2309 let mut expected = 0u64;
2310 for output in &outputs {
2311 let m = std::fs::metadata(output).unwrap();
2312 let inode = (m.dev(), m.ino());
2313 if inode != retained_inode && seen.insert(inode) {
2314 expected += m.blocks() * 512;
2315 }
2316 }
2317 assert!(expected > 0);
2318 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2319 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2320 assert_eq!(report.freed_bytes, logical, "{report:?}");
2321 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2322 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2323 assert!(outputs.iter().all(|p| !p.exists()));
2324 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2325 assert!(wt.join("src/lib.rs").is_file());
2326 }
2327
2328 #[tokio::test]
2329 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2330 let f = Fx::new().await;
2331 let state = f.run("ca03", true).await;
2332 let wt = &state.candidates[0].worktree;
2333 Fx::target(wt);
2334 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2335 let unknown = wt.join("target/debug/operator-note");
2336 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2337 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2338 assert_eq!(report.proven_reclaimable_bytes, 0);
2339 #[cfg(unix)]
2340 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2341 assert!(report.protected_or_unproven_bytes >= 4096);
2342 let message = || {
2343 crate::notices::Notices::at(f.home.join("notifications"))
2344 .list()
2345 .into_iter()
2346 .find(|n| n.key == "disk:retention")
2347 .unwrap()
2348 .message
2349 };
2350 let notice = message();
2351 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2352 assert!(notice.contains("protected or unproven"), "{notice}");
2353 assert!(
2354 notice.contains("legacy incremental/build and unknown files retained"),
2355 "{notice}"
2356 );
2357 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2358 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2359 assert_eq!(report.freed_bytes, 0);
2360 let notice = message();
2361 assert!(notice.contains(&state.id), "{notice}");
2362 assert!(
2363 notice.contains("lock") || notice.contains("owner"),
2364 "{notice}"
2365 );
2366 drop(owner);
2367 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2368 }
2369
2370 #[tokio::test]
2371 async fn real_cargo_build_holds_the_target_against_reclamation() {
2372 let f = Fx::new().await;
2373 let state = f.run("ca02", true).await;
2374 let wt = &state.candidates[0].worktree;
2375 Fx::target(wt);
2376 std::fs::create_dir(wt.join("src")).unwrap();
2377 std::fs::write(
2378 wt.join("Cargo.toml"),
2379 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2380 )
2381 .unwrap();
2382 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2383 std::fs::write(
2384 wt.join("build.rs"),
2385 r#"fn main() {
2386 println!("cargo:rerun-if-env-changed=FIXTURE_READY");
2387 let Ok(ready) = std::env::var("FIXTURE_READY") else { return };
2388 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2389 std::fs::write(ready, "ready").unwrap();
2390 let start = std::time::Instant::now();
2391 while !std::path::Path::new(&release).exists() {
2392 assert!(start.elapsed().as_secs() < 120, "fixture was not released");
2393 std::thread::sleep(std::time::Duration::from_millis(10));
2394 }
2395 }"#,
2396 )
2397 .unwrap();
2398 let ready = wt.join("ready");
2399 let release = wt.join("release");
2400 let mut cargo = tokio::process::Command::new("cargo");
2401 cargo.env_clear();
2402 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2403 if let Some(value) = std::env::var_os(key) {
2404 cargo.env(key, value);
2405 }
2406 }
2407 cargo
2408 .current_dir(wt)
2409 .env("RUSTUP_TOOLCHAIN", "stable")
2410 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2411 .env("CARGO_INCREMENTAL", "1")
2412 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2413 .arg(wt.join("target"));
2414 let warm = cargo.output().await.unwrap();
2418 assert!(
2419 warm.status.success(),
2420 "fixture compilation failed: {}",
2421 String::from_utf8_lossy(&warm.stderr)
2422 );
2423 let log_path = f._temp.path().join("live-cargo.stderr");
2424 let mut child = cargo
2425 .env("FIXTURE_READY", &ready)
2426 .env("FIXTURE_RELEASE", &release)
2427 .stdout(std::process::Stdio::null())
2428 .stderr(std::fs::File::create(&log_path).unwrap())
2429 .kill_on_drop(true)
2430 .spawn()
2431 .unwrap();
2432 let readiness = tokio::time::timeout(std::time::Duration::from_secs(60), async {
2433 while !ready.exists() {
2434 if let Some(status) = child.try_wait().unwrap() {
2435 return Err(format!("fixture exited before readiness: {status}"));
2436 }
2437 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2438 }
2439 Ok(())
2440 })
2441 .await;
2442 let arrived = matches!(readiness, Ok(Ok(())));
2443 let report = if arrived {
2444 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2445 } else {
2446 None
2447 };
2448 std::fs::write(release, "release").unwrap();
2450 let status = tokio::time::timeout(std::time::Duration::from_secs(60), child.wait()).await;
2451 if status.is_err() {
2452 child.kill().await.unwrap();
2453 child.wait().await.unwrap();
2454 }
2455 assert!(
2456 arrived,
2457 "fixture build never reached its active build script: {readiness:?}\n{}",
2458 std::fs::read_to_string(&log_path).unwrap()
2459 );
2460 assert!(
2461 status.unwrap().unwrap().success(),
2462 "fixture build failed: {}",
2463 std::fs::read_to_string(&log_path).unwrap()
2464 );
2465 let report = report.unwrap();
2466 assert_eq!(report.freed_bytes, 0, "{report:?}");
2467 assert!(
2468 report
2469 .retained
2470 .iter()
2471 .any(|s| s.contains("Cargo build is active")),
2472 "{report:?}"
2473 );
2474 assert!(wt.join("target/debug/output").is_file());
2475 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2476 }
2477
2478 #[cfg(unix)]
2479 #[test]
2480 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2481 let temp = tempfile::tempdir().unwrap();
2482 let a = temp.path().join("a");
2483 let b = temp.path().join("b");
2484 let outside = temp.path().join("retained-input");
2485 std::fs::write(&a, vec![1; 65536]).unwrap();
2486 std::fs::hard_link(&a, &b).unwrap();
2487 std::fs::hard_link(&a, &outside).unwrap();
2488 let mut seen = Default::default();
2489 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2490 assert!(blocks > 0);
2491 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2492 std::fs::remove_file(&a).unwrap();
2493 assert_eq!(accounting.unlinked(&a), Some(0));
2494 std::fs::remove_file(&b).unwrap();
2495 assert_eq!(accounting.unlinked(&b), Some(0));
2496 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2497 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2498 std::fs::remove_file(&outside).unwrap();
2499 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2500 }
2501
2502 #[cfg(unix)]
2503 #[test]
2504 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2505 let temp = tempfile::tempdir_in(".").unwrap();
2506 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2507 assert!(home.is_relative());
2508 let paths = vec![std::env::current_dir().unwrap()];
2509 let gib = 1024 * 1024 * 1024;
2510 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2511 assert!(
2512 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2513 9 * gib
2514 ))
2515 .is_err()
2516 );
2517 drop(reservation);
2518 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2519 let bad = temp.path().join("symlink-state");
2520 std::fs::create_dir(&bad).unwrap();
2521 std::os::unix::fs::symlink(
2522 home.canonicalize().unwrap().join("disk-admission"),
2523 bad.join("disk-admission"),
2524 )
2525 .unwrap();
2526 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2527 }
2528
2529 #[cfg(unix)]
2530 #[test]
2531 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2532 let temp = tempfile::tempdir().unwrap();
2533 let home = temp.path().canonicalize().unwrap();
2534 let paths = vec![home.clone(), home.join("not-created-yet")];
2535 let gib = 1024 * 1024 * 1024;
2536 let floor = 8 * gib;
2537 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2538 assert_eq!(first.len(), 1);
2539 let blocked =
2540 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2541 assert!(blocked.to_string().contains("active writer reservations"));
2542 let second =
2543 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2544 drop(first);
2545 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2546 drop(second);
2547 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2548 assert!(
2549 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2550 );
2551 }
2552
2553 #[cfg(unix)]
2554 #[test]
2555 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2556 let temp = tempfile::tempdir().unwrap();
2557 let home = temp.path().canonicalize().unwrap();
2558 let barrier = std::sync::Barrier::new(2);
2559 let gib = 1024 * 1024 * 1024;
2560 let results = std::thread::scope(|scope| {
2561 let threads: Vec<_> = (0..2)
2562 .map(|_| {
2563 scope.spawn(|| {
2564 barrier.wait();
2565 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2566 Ok(100 + gib)
2567 })
2568 })
2569 })
2570 .collect();
2571 threads
2572 .into_iter()
2573 .map(|thread| thread.join().unwrap())
2574 .collect::<Vec<_>>()
2575 });
2576 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2577 assert!(
2578 results
2579 .iter()
2580 .filter_map(|r| r.as_ref().err())
2581 .all(|e| e.to_string().contains("active writer reservations"))
2582 );
2583 }
2584
2585 #[tokio::test]
2586 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2587 real_cargo_fixture(false).await;
2588 }
2589
2590 #[tokio::test]
2591 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2592 real_cargo_fixture(true).await;
2593 }
2594
2595 async fn real_cargo_fixture(precreate: bool) {
2596 let f = Fx::new().await;
2597 let state = f.run("ca00", true).await;
2598 let wt = &state.candidates[0].worktree;
2599 let source = "fn main() { println!(\"retained fixture\"); }\n";
2600 std::fs::create_dir(wt.join("src")).unwrap();
2601 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2602 std::fs::write(
2603 wt.join("Cargo.toml"),
2604 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2605 )
2606 .unwrap();
2607 if precreate {
2608 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2609 }
2610 let mut cargo = std::process::Command::new("cargo");
2611 cargo.env_clear();
2612 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2613 if let Some(value) = std::env::var_os(key) {
2614 cargo.env(key, value);
2615 }
2616 }
2617 let build = cargo
2618 .current_dir(wt)
2619 .env("RUSTUP_TOOLCHAIN", "stable")
2620 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2621 .env("CARGO_INCREMENTAL", "0")
2622 .arg("--config")
2623 .arg(format!(
2624 "build.build-dir={}",
2625 serde_json::to_string(&wt.join("target")).unwrap()
2626 ))
2627 .args([
2628 "build",
2629 "--offline",
2630 "--release",
2631 "--jobs",
2632 "1",
2633 "--target-dir",
2634 ])
2635 .arg(wt.join("target"))
2636 .output()
2637 .unwrap();
2638 assert!(
2639 build.status.success(),
2640 "{}",
2641 String::from_utf8_lossy(&build.stderr)
2642 );
2643 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2644 if precreate {
2645 let again = cargo.output().unwrap();
2647 assert!(
2648 again.status.success(),
2649 "{}",
2650 String::from_utf8_lossy(&again.stderr)
2651 );
2652 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2653 assert!(cargo_target(wt).await.is_err());
2654 }
2655 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2656 .await
2657 .unwrap();
2658 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2659 .await
2660 .unwrap();
2661 let artifact = wt.join("target/release/retained-fixture");
2662 assert!(artifact.is_file());
2663 let unknown = wt.join("target/release/operator-note");
2664 std::fs::write(&unknown, "keep this note\n").unwrap();
2665
2666 let target = wt.join("target");
2667 let mut preserved = std::collections::BTreeMap::new();
2668 for relative in [
2669 "tmp/sentinel",
2670 "magi-tools/sentinel",
2671 "lifecycle-client-check.log",
2672 ] {
2673 let path = target.join(relative);
2674 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2675 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2676 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2677 }
2678 let decoy = target.join("magi-tools/retained-fixture");
2681 std::fs::copy(&artifact, &decoy).unwrap();
2682 let dep_info = decoy.with_extension("d");
2683 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2684 for path in [
2685 decoy,
2686 dep_info,
2687 wt.join("Cargo.toml"),
2688 wt.join("Cargo.lock"),
2689 ] {
2690 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2691 }
2692 let supported_dirs = profile_output_dirs(&target.join("release"), true).unwrap();
2693 let proven = declared_outputs(&target).unwrap();
2694 assert!(
2695 proven
2696 .iter()
2697 .any(|path| path.parent() != Some(target.join("release").as_path())),
2698 "both uplifted and unit/deps binary references must be proven: {proven:?}"
2699 );
2700
2701 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2702 assert!(out.freed_bytes > 0, "{out:?}");
2703 for (path, bytes) in preserved {
2704 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2705 }
2706 assert!(!artifact.exists(), "{out:?}");
2707 assert_eq!(
2708 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2709 source
2710 );
2711 assert_eq!(
2712 std::fs::read_to_string(&unknown).unwrap(),
2713 "keep this note\n"
2714 );
2715 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2716 let ledger: serde_json::Value = serde_json::from_slice(
2717 &std::fs::read(
2718 f.home
2719 .join("runs")
2720 .join(&state.id)
2721 .join("build-outputs.json"),
2722 )
2723 .unwrap(),
2724 )
2725 .unwrap();
2726 assert_eq!(
2727 ledger["identity"],
2728 if precreate {
2729 "rustc-profile-dep-info"
2730 } else {
2731 "cache-tag"
2732 }
2733 );
2734 if precreate {
2735 assert!(!target.join("CACHEDIR.TAG").exists());
2736 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2737 }
2738 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2739 assert!(deleted.contains(&artifact));
2740 assert_eq!(deleted, proven);
2741 for path in deleted {
2742 assert!(!path.exists());
2743 assert!(supported_dirs.contains(&path.parent().unwrap().to_owned()));
2744 }
2745 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2746 }
2747
2748 fn missing_tag_target(wt: &Path) {
2749 Fx::target(wt);
2750 let target = wt.join("target");
2751 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2752 for subdir in ["debug/deps", "debug/.fingerprint"] {
2753 std::fs::create_dir_all(target.join(subdir)).unwrap();
2754 }
2755 std::fs::write(target.join(".rustc_info.json"),
2756 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2757 }
2758
2759 fn v2_target(wt: &Path) -> PathBuf {
2761 missing_tag_target(wt);
2762 let target = wt.join("target");
2763 std::fs::remove_dir(target.join("debug/deps")).unwrap();
2764 std::fs::remove_dir(target.join("debug/.fingerprint")).unwrap();
2765 let unit = target.join("debug/build/fixture/0123456789abcdef");
2766 let out = unit.join("out");
2767 std::fs::create_dir_all(&out).unwrap();
2768 std::fs::create_dir(unit.join("fingerprint")).unwrap();
2769 for name in [
2770 "lib-fixture",
2771 "lib-fixture.json",
2772 "dep-lib-fixture",
2773 "invoked.timestamp",
2774 ] {
2775 std::fs::write(unit.join("fingerprint").join(name), "fixture").unwrap();
2776 }
2777 let artifact = out.join("libfixture-0123456789abcdef.rlib");
2778 std::fs::copy(target.join("debug/output"), &artifact).unwrap();
2779 std::fs::write(
2780 out.join("fixture-0123456789abcdef.d"),
2781 format!("{}: source\n", artifact.display()),
2782 )
2783 .unwrap();
2784 artifact
2785 }
2786
2787 #[tokio::test]
2788 async fn missing_tag_v2_proves_only_compiler_units_and_preserves_retained_links() {
2789 let f = Fx::new().await;
2790 let state = f.run("bc01", true).await;
2791 let wt = &state.candidates[0].worktree;
2792 let artifact = v2_target(wt);
2793 let unknown = artifact.parent().unwrap().join("operator-note");
2794 std::fs::write(&unknown, "retain").unwrap();
2795 let script_unit = wt.join("target/debug/build/fixture/fedcba9876543210");
2797 let script = script_unit.join("out");
2798 std::fs::create_dir_all(&script).unwrap();
2799 std::fs::create_dir(script_unit.join("fingerprint")).unwrap();
2800 std::fs::write(script_unit.join("fingerprint/invoked.timestamp"), "fixture").unwrap();
2801 let decoy = script.join("libfixture-fedcba9876543210.rlib");
2802 std::fs::copy(&artifact, &decoy).unwrap();
2803 std::fs::write(
2804 script.join("fixture-fedcba9876543210.d"),
2805 format!("{}: source\n", decoy.display()),
2806 )
2807 .unwrap();
2808 let uplift = wt.join("target/debug/libfixture.rlib");
2810 std::fs::hard_link(&artifact, &uplift).unwrap();
2811 std::fs::write(
2812 wt.join("target/debug/fixture.d"),
2813 format!("{}: source\n", uplift.display()),
2814 )
2815 .unwrap();
2816 let bin_unit = wt.join("target/debug/build/fixture/1111111111111111");
2817 std::fs::create_dir_all(bin_unit.join("out")).unwrap();
2818 std::fs::create_dir(bin_unit.join("fingerprint")).unwrap();
2819 for name in [
2820 "bin-fixture",
2821 "bin-fixture.json",
2822 "dep-bin-fixture",
2823 "invoked.timestamp",
2824 ] {
2825 std::fs::write(bin_unit.join("fingerprint").join(name), "fixture").unwrap();
2826 }
2827 let bin = bin_unit.join("out/fixture");
2828 std::fs::hard_link(wt.join("target/debug/output"), &bin).unwrap();
2829 std::fs::write(
2830 bin.with_extension("d"),
2831 format!("{}: source\n", bin.display()),
2832 )
2833 .unwrap();
2834 let retained = wt.join("retained-link");
2835 std::fs::hard_link(&artifact, &retained).unwrap();
2836 use std::os::unix::fs::MetadataExt;
2837 let expected_allocation = std::fs::metadata(wt.join("target/debug/output"))
2838 .unwrap()
2839 .blocks()
2840 * 512;
2841 let sentinel = std::fs::read(&retained).unwrap();
2842 let first = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2843 assert_eq!(first.freed_bytes, 800, "{first:?}");
2844 assert!(!bin.exists());
2845 assert_eq!(
2846 first.estimated_released_allocated_bytes,
2847 Some(expected_allocation)
2848 );
2849 assert_eq!(std::fs::read(&retained).unwrap(), sentinel);
2850 assert!(retained.is_file());
2851 assert!(unknown.is_file());
2852 assert!(decoy.is_file());
2853 assert!(!artifact.exists());
2854 assert_eq!(std::fs::metadata(retained).unwrap().len(), 200);
2856 }
2857
2858 #[tokio::test]
2859 async fn missing_tag_v2_accepts_completed_build_script_run_units() {
2860 let f = Fx::new().await;
2861 let state = f.run("bc02", true).await;
2862 let wt = &state.candidates[0].worktree;
2863 let artifact = v2_target(wt);
2864 let run = wt.join("target/debug/build/fixture/2222222222222222");
2865 std::fs::create_dir_all(run.join("out")).unwrap();
2866 std::fs::create_dir(run.join("fingerprint")).unwrap();
2867 for name in [
2868 "run-build-script-build-script-build",
2869 "run-build-script-build-script-build.json",
2870 ] {
2871 std::fs::write(run.join("fingerprint").join(name), "fixture").unwrap();
2872 }
2873 let decoy = run.join("out/libdecoy-2222222222222222.rlib");
2875 std::fs::copy(&artifact, &decoy).unwrap();
2876 std::fs::write(
2877 run.join("out/decoy-2222222222222222.d"),
2878 format!("{}: source\n", decoy.display()),
2879 )
2880 .unwrap();
2881 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2882 assert!(report.freed_bytes > 0, "{report:?}");
2883 assert!(!artifact.exists());
2884 assert!(decoy.is_file());
2885 }
2886
2887 #[tokio::test]
2888 async fn missing_tag_v2_rejects_incomplete_mixed_and_symlinked_layouts() {
2889 for invalid in [
2890 "build",
2891 "deps",
2892 ".fingerprint",
2893 "out",
2894 "fingerprint",
2895 "hash",
2896 "mixed",
2897 "unit-fingerprint",
2898 "compiler-fingerprint",
2899 "absent-out",
2900 ] {
2901 let f = Fx::new().await;
2902 let state = f.run("bc02", true).await;
2903 let wt = &state.candidates[0].worktree;
2904 let artifact = v2_target(wt);
2905 let unit = artifact.parent().unwrap().parent().unwrap();
2906 let profile = wt.join("target/debug");
2907 match invalid {
2908 "mixed" => {
2909 std::fs::create_dir(profile.join("deps")).unwrap();
2910 std::fs::create_dir(profile.join(".fingerprint")).unwrap();
2911 }
2912 "hash" => {
2913 std::fs::rename(unit, unit.with_file_name("unknown")).unwrap();
2914 }
2915 "unit-fingerprint" => {
2916 std::fs::remove_file(unit.join("fingerprint/invoked.timestamp")).unwrap();
2917 }
2918 "compiler-fingerprint" => {
2919 std::fs::remove_file(unit.join("fingerprint/lib-fixture.json")).unwrap();
2920 }
2921 "absent-out" => {
2922 std::fs::remove_dir_all(unit.join("out")).unwrap();
2923 }
2924 name => {
2925 let path = if matches!(name, "out" | "fingerprint") {
2926 unit.join(name)
2927 } else {
2928 profile.join(name)
2929 };
2930 if path.exists() {
2931 std::fs::remove_dir_all(&path).unwrap();
2932 }
2933 std::os::unix::fs::symlink(&f.repo, &path).unwrap();
2934 }
2935 }
2936 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2937 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2938 assert!(!report.retained.is_empty(), "{invalid}: {report:?}");
2939 }
2940 }
2941
2942 #[tokio::test]
2943 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2944 for legacy in [false, true] {
2945 let f = Fx::new().await;
2946 let mut state = f.run("ac00", true).await;
2947 Fx::target(&state.candidates[0].worktree);
2948 let original = state.updated_at;
2949 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2950 if legacy {
2951 let path = f.home.join("runs").join(&state.id).join("run.json");
2952 let mut value = serde_json::to_value(&state).unwrap();
2953 value.as_object_mut().unwrap().remove("output_activity_at");
2954 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2955 state = RunState::load_under(&state.id, &f.home).unwrap();
2956 assert_eq!(state.output_activity(), original);
2957 }
2958 let age = state.output_activity();
2959 for offset in [30, 20, 10] {
2960 state
2961 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2962 .unwrap();
2963 state = RunState::load_under(&state.id, &f.home).unwrap();
2964 assert_eq!(state.output_activity(), age);
2965 }
2966 let mut disk = Fx::disk();
2967 disk.fold_grace_secs = 3600;
2968 let report = sweep(&f.home, &disk, now).await;
2969 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2970 }
2971 }
2972
2973 #[tokio::test]
2974 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2975 for evidence in [
2976 "execution",
2977 "artifact",
2978 "dep-info",
2979 "fingerprint",
2980 "lease",
2981 "future",
2982 ] {
2983 let f = Fx::new().await;
2984 let mut state = f.run("ac01", true).await;
2985 let wt = state.candidates[0].worktree.clone();
2986 missing_tag_target(&wt);
2987 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2988 let recent = now - jiff::SignedDuration::from_secs(10);
2989 match evidence {
2990 "execution" => {
2991 state.note_output_activity(recent);
2992 state.save_under_at(&f.home, recent).unwrap();
2993 }
2994 "lease" => {
2995 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2996 let file = lease._file.try_clone().unwrap();
2997 drop(lease);
2998 file.set_modified(std::time::SystemTime::from(recent))
2999 .unwrap();
3000 }
3001 _ => {
3002 let relative = match evidence {
3003 "dep-info" => "debug/output.d",
3004 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
3005 _ => "debug/output",
3006 };
3007 let path = wt.join("target").join(relative);
3008 let file = OpenOptions::new()
3009 .write(true)
3010 .create(true)
3011 .truncate(false)
3012 .open(path)
3013 .unwrap();
3014 let at = if evidence == "future" {
3015 now + jiff::SignedDuration::from_secs(10)
3016 } else {
3017 recent
3018 };
3019 file.set_modified(std::time::SystemTime::from(at)).unwrap();
3020 }
3021 }
3022 let mut disk = Fx::disk();
3023 disk.fold_grace_secs = 3600;
3024 let report = sweep(&f.home, &disk, now).await;
3025 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
3026 assert!(
3027 report.retained.iter().any(|s| s.contains("grace period")
3028 || s.contains("execution activity after inventory")),
3029 "{evidence}: {report:?}"
3030 );
3031 let aged = now + jiff::SignedDuration::from_secs(7200);
3032 let report = sweep(&f.home, &disk, aged).await;
3033 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
3034 }
3035 }
3036
3037 #[tokio::test]
3038 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
3039 let f = Fx::new().await;
3040 let state = f.run("ac02", true).await;
3041 let wt = &state.candidates[0].worktree;
3042 missing_tag_target(wt);
3043 let target = wt.join("target");
3044 let unknown = target.join("debug/notes");
3045 std::fs::write(&unknown, "operator data").unwrap();
3046 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
3047 let path = target.join("debug").join(lock);
3048 let file = OpenOptions::new()
3049 .create(true)
3050 .truncate(false)
3051 .read(true)
3052 .write(true)
3053 .open(&path)
3054 .unwrap();
3055 crate::pr_head::try_lock_file(&file).unwrap();
3056 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3057 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
3058 assert!(
3059 report
3060 .retained
3061 .iter()
3062 .any(|s| s.contains(path.to_str().unwrap())),
3063 "{report:?}"
3064 );
3065 }
3066 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
3067 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3068 drop(owner);
3069 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
3070 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3071 drop(driver);
3072 let output = OutputLease::borrow(wt).unwrap().unwrap();
3073 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3074 drop(output);
3075 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3076 assert_eq!(report.freed_bytes, 200, "{report:?}");
3077 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
3078 assert!(!target.join("CACHEDIR.TAG").exists());
3079 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
3080 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
3081 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3082 }
3083
3084 #[tokio::test]
3085 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
3086 let f = Fx::new().await;
3087 let state = f.run("ac04", true).await;
3088 let wt = &state.candidates[0].worktree;
3089 missing_tag_target(wt);
3090 let target = wt.join("target");
3091 let first = target.join("debug/output");
3092 let second = target.join("debug/second");
3093 std::fs::copy(&first, &second).unwrap();
3094 std::fs::write(
3095 second.with_extension("d"),
3096 format!("{}: source\n", second.display()),
3097 )
3098 .unwrap();
3099 let outputs = declared_outputs(&target).unwrap();
3100 assert_eq!(outputs.len(), 2);
3101 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
3103 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
3104 std::fs::remove_file(first).unwrap();
3105 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3106 assert_eq!(report.freed_bytes, 200, "{report:?}");
3107 assert!(!second.exists());
3108 assert!(second.with_extension("d").exists());
3109 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
3110 }
3111
3112 #[tokio::test]
3113 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
3114 for invalid in [
3115 "identity",
3116 "deps",
3117 "fingerprint",
3118 "lock",
3119 "dep-info",
3120 "tag-symlink",
3121 "bad-tag",
3122 ] {
3123 let f = Fx::new().await;
3124 let state = f.run("ac03", true).await;
3125 let wt = &state.candidates[0].worktree;
3126 missing_tag_target(wt);
3127 let target = wt.join("target");
3128 match invalid {
3129 "identity" => std::fs::write(
3130 target.join(".rustc_info.json"),
3131 r#"{"rustc_fingerprint":123}"#,
3132 )
3133 .unwrap(),
3134 "deps" | "fingerprint" => {
3135 let path = target.join("debug").join(if invalid == "deps" {
3136 "deps"
3137 } else {
3138 ".fingerprint"
3139 });
3140 std::fs::remove_dir(&path).unwrap();
3141 std::os::unix::fs::symlink(&f.repo, path).unwrap();
3142 }
3143 "lock" => {
3144 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
3145 }
3146 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
3147 "tag-symlink" => {
3148 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
3149 .unwrap()
3150 }
3151 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
3152 _ => unreachable!(),
3153 }
3154 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3155 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
3156 assert!(target.join("debug/output").is_file());
3157 }
3158 }
3159
3160 #[tokio::test]
3161 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
3162 let f = Fx::new().await;
3163 let first = f.run("ba00", true).await;
3164 let second = f.run("ba01", true).await;
3165 for state in [&first, &second] {
3166 Fx::target(&state.candidates[0].worktree);
3167 }
3168 let mut disk = Fx::disk();
3169 disk.worktree_reclaim_bytes = 100;
3170 let out = sweep(&f.home, &disk, Fx::now()).await;
3171 assert_eq!(out.freed_bytes, 200, "{out:?}");
3172 assert!(
3173 out.retained
3174 .contains(&format!("{}: per-pass byte budget spent", second.id)),
3175 "{out:?}"
3176 );
3177 assert!(
3178 !first.candidates[0]
3179 .worktree
3180 .join("target/debug/output")
3181 .exists()
3182 );
3183 assert!(
3184 second.candidates[0]
3185 .worktree
3186 .join("target/debug/output")
3187 .exists()
3188 );
3189 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
3190 }
3191
3192 #[tokio::test]
3193 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
3194 let f = Fx::new().await;
3195 let small = f.run("bb00", true).await;
3196 let large = f.run("bb01", true).await;
3197 for state in [&small, &large] {
3198 Fx::target(&state.candidates[0].worktree);
3199 }
3200 let output = large.candidates[0].worktree.join("target/debug/output");
3201 std::fs::OpenOptions::new()
3202 .write(true)
3203 .open(&output)
3204 .unwrap()
3205 .set_len(1000)
3206 .unwrap();
3207 let mut disk = Fx::disk();
3208 disk.worktree_reclaim_bytes = 300;
3209 disk.min_free_bytes = u64::MAX;
3210 disk.worktree_limit_bytes = u64::MAX;
3212 let out = sweep(&f.home, &disk, Fx::now()).await;
3213 assert_eq!(out.freed_bytes, 1000, "{out:?}");
3214 assert!(!output.exists());
3215 assert!(
3216 small.candidates[0]
3217 .worktree
3218 .join("target/debug/output")
3219 .exists()
3220 );
3221 }
3222
3223 #[tokio::test]
3224 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
3225 let f = Fx::new().await;
3226 let first = f.run("bc00", true).await;
3227 let second = f.run("bc01", true).await;
3228 for state in [&first, &second] {
3229 Fx::target(&state.candidates[0].worktree);
3230 }
3231 std::fs::create_dir_all(
3233 first.candidates[0]
3234 .worktree
3235 .join("target/debug/incremental"),
3236 )
3237 .unwrap();
3238 std::fs::write(
3239 first.candidates[0]
3240 .worktree
3241 .join("target/debug/incremental/cache"),
3242 [0; 1000],
3243 )
3244 .unwrap();
3245 let mut disk = Fx::disk();
3246 disk.worktree_reclaim_bytes = 0;
3247 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3248 disk.worktree_reclaim_bytes = 300;
3249 let out = sweep(&f.home, &disk, Fx::now()).await;
3250 assert_eq!(out.freed_bytes, 200, "{out:?}");
3251 assert!(
3252 !first.candidates[0]
3253 .worktree
3254 .join("target/debug/output")
3255 .exists()
3256 );
3257 assert!(
3258 second.candidates[0]
3259 .worktree
3260 .join("target/debug/output")
3261 .exists()
3262 );
3263 assert!(
3264 out.retained
3265 .iter()
3266 .any(|r| r.contains("declared outputs exceed"))
3267 );
3268 }
3269
3270 #[tokio::test]
3271 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
3272 let f = Fx::new().await;
3273 let mut protected = f.run("bd00", true).await;
3274 let later = f.run("bd01", true).await;
3275 for state in [&protected, &later] {
3276 Fx::target(&state.candidates[0].worktree);
3277 }
3278 let mut disk = Fx::disk();
3279 disk.worktree_reclaim_bytes = 100;
3280 disk.fold_grace_secs = 3600;
3281 let out = sweep(&f.home, &disk, Fx::now()).await;
3282 assert_eq!(out.freed_bytes, 0);
3283 assert!(
3284 out.retained
3285 .iter()
3286 .any(|r| r.contains("within grace period"))
3287 );
3288 disk.fold_grace_secs = 0;
3289 protected.driver_exited = false;
3290 protected.driver_pid = Some(std::process::id());
3291 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
3292 protected
3293 .save_under_at(&f.home, protected.updated_at)
3294 .unwrap();
3295 let out = sweep(&f.home, &disk, Fx::now()).await;
3296 assert!(
3297 out.retained
3298 .iter()
3299 .any(|r| r.contains("run live or ownership unknown"))
3300 );
3301 assert_eq!(
3302 out.freed_bytes, 200,
3303 "live target must not consume exception: {out:?}"
3304 );
3305 let output = protected.candidates[0].worktree.join("target/debug/output");
3306 assert!(output.exists());
3307 protected.driver_exited = true;
3308 protected.save_under(&f.home).unwrap();
3309 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
3310 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3311 assert!(output.exists());
3312 drop(owner);
3313 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
3314 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3315 assert!(output.exists());
3316 drop(cargo);
3317 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
3318 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3319 assert!(output.exists());
3320 drop(lease);
3321 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
3322 assert!(!output.exists());
3323 }
3324
3325 #[tokio::test]
3326 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
3327 for remove_root in [true, false] {
3328 let f = Fx::new().await;
3329 let stale = Fx::new().await;
3330 let mut missing = stale.run("aa00", false).await;
3331 Fx::target(&missing.candidates[0].worktree);
3332 missing.save_under(&f.home).unwrap();
3333 if remove_root {
3334 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
3335 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
3338 } else {
3339 std::fs::remove_dir_all(&stale.repo).unwrap();
3340 }
3341 let healthy = f.run("aa01", false).await;
3342 Fx::target(&healthy.candidates[0].worktree);
3343
3344 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3345 assert!(report.failures.is_empty(), "{report:?}");
3346 assert_eq!(report.freed_bytes, 200, "{report:?}");
3347 assert!(
3348 !healthy.candidates[0]
3349 .worktree
3350 .join("target/debug/output")
3351 .exists()
3352 );
3353 if !remove_root {
3354 assert!(
3356 missing.candidates[0]
3357 .worktree
3358 .join("target/debug/output")
3359 .exists()
3360 );
3361 }
3362 }
3363 }
3364
3365 #[tokio::test]
3366 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
3367 let f = Fx::new().await;
3368 let mut missing = f.run("aa00", false).await;
3369 Fx::target(&missing.candidates[0].worktree);
3370 missing.repo = missing.candidates[0].worktree.clone();
3371 missing.save_under(&f.home).unwrap();
3372 std::fs::remove_dir_all(&missing.repo).unwrap();
3373 assert!(missing.worktree_root().is_dir());
3374 assert!(f.repo.is_dir());
3375
3376 let healthy = f.run("aa01", false).await;
3377 Fx::target(&healthy.candidates[0].worktree);
3378
3379 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3380 assert!(report.failures.is_empty(), "{report:?}");
3381 assert_eq!(report.freed_bytes, 200, "{report:?}");
3382 assert!(
3383 !healthy.candidates[0]
3384 .worktree
3385 .join("target/debug/output")
3386 .exists()
3387 );
3388 }
3389
3390 #[tokio::test]
3391 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
3392 let f = Fx::new().await;
3393 let broken = Fx::new().await;
3394 let mut invalid = broken.run("aa00", false).await;
3395 Fx::target(&invalid.candidates[0].worktree);
3396 invalid.save_under(&f.home).unwrap();
3397 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
3398 let healthy = f.run("aa01", false).await;
3399 Fx::target(&healthy.candidates[0].worktree);
3400
3401 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3402 assert_eq!(report.failures.len(), 1, "{report:?}");
3403 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
3404 assert_eq!(report.freed_bytes, 200, "{report:?}");
3405 assert!(
3406 !healthy.candidates[0]
3407 .worktree
3408 .join("target/debug/output")
3409 .exists()
3410 );
3411 assert!(
3412 invalid.candidates[0]
3413 .worktree
3414 .join("target/debug/output")
3415 .exists()
3416 );
3417 }
3418
3419 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
3420 let path = reconcile::run(&f.home, &task.id, &source.id, None)
3421 .await
3422 .unwrap();
3423 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
3424 }
3425
3426 #[tokio::test]
3427 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
3428 use crate::{
3429 chat_control::{Operation, dispatch},
3430 talk::{self, Talks},
3431 };
3432 let f = Fx::new().await;
3433 let source = f.run("ae00", true).await;
3434 let clean = f.run("ae01", false).await;
3435 let queue = crate::queue::Queue::at(f.home.join("queue"));
3436 let talks = Talks::at(f.home.join("talks"));
3437 let open_talk = |id: &str| -> talk::Talk {
3438 let mut conversation = serde_json::from_value(serde_json::json!({
3439 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
3440 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
3441 "seat":crate::agent::SeatState::new("talk","worker",1)
3442 }))
3443 .unwrap();
3444 talks.put(&mut conversation).unwrap();
3445 conversation
3446 };
3447 let conversation = open_talk("talk-owner");
3448 let other = open_talk("talk-other");
3449 let task = f.task(&[source.clone(), clean.clone()]);
3451 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3452 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
3453 task: task.id.clone(),
3454 authority: source.id.clone(),
3455 apply: apply.map(str::to_owned),
3456 };
3457 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
3459 .await
3460 .unwrap();
3461 let legacy: serde_json::Value =
3462 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
3463 let legacy_id = legacy["id"].as_str().unwrap();
3464 assert!(
3465 dispatch(
3466 &queue,
3467 &f.home,
3468 &talks,
3469 &conversation.id,
3470 operation(Some(legacy_id))
3471 )
3472 .await
3473 .is_err()
3474 );
3475 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
3476 .await
3477 .unwrap();
3478 assert!(summary.contains("Complete review summary"));
3479 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
3480 let root = f.home.join("retention-reconciliations");
3481 let displayed: serde_json::Value =
3482 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
3483 let id = displayed["plan"].as_str().unwrap();
3484 let dir = root.join(id);
3485 let plan: serde_json::Value =
3486 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
3487 assert_eq!(plan["version"], 2);
3488 assert!(clean.candidates[0].worktree.exists());
3489 assert!(!dir.join("approved.json").exists());
3490 assert!(
3492 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
3493 .await
3494 .is_err()
3495 );
3496 assert!(!dir.join("approved.json").exists());
3497 let error = reconcile::interrupt_chat_after_fence(
3500 &f.home,
3501 &task,
3502 id,
3503 reconcile::ChatRequest {
3504 talks: &talks,
3505 conversation: &conversation.id,
3506 },
3507 )
3508 .await
3509 .unwrap_err();
3510 assert!(
3511 error.to_string().contains("simulated interruption"),
3512 "{error:#}"
3513 );
3514 assert!(clean.candidates[0].worktree.exists());
3515 assert!(dir.join("approved.json").exists());
3516 dispatch(
3517 &queue,
3518 &f.home,
3519 &talks,
3520 &conversation.id,
3521 operation(Some(id)),
3522 )
3523 .await
3524 .unwrap();
3525 assert!(!clean.candidates[0].worktree.exists());
3526 assert!(source.candidates[0].worktree.exists());
3527 let audit: serde_json::Value =
3528 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3529 assert_eq!(audit["version"], 2);
3530 assert_eq!(
3531 audit["owner"],
3532 serde_json::json!({"conversation": conversation.id})
3533 );
3534 dispatch(
3535 &queue,
3536 &f.home,
3537 &talks,
3538 &conversation.id,
3539 operation(Some(id)),
3540 )
3541 .await
3542 .unwrap();
3543 }
3544
3545 #[tokio::test]
3546 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3547 let f = Fx::new().await;
3548 let source = f.run("af00", true).await;
3549 let mut clean = f.run("af01", false).await;
3550 let mut dirty = f.run("af02", false).await;
3551 for state in [&mut clean, &mut dirty] {
3552 state.status = RunStatus::Prep;
3553 state.prep_complete = Some(false);
3554 state.candidates[0].empty = false;
3555 state.save_under(&f.home).unwrap();
3556 }
3557 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3558 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3559 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3560 let plan = reconciliation_plan(&f, &task, &source).await;
3561 let selected: Vec<_> = plan["candidates"]
3562 .as_array()
3563 .unwrap()
3564 .iter()
3565 .filter(|e| e["selected"] == true)
3566 .collect();
3567 assert_eq!(selected.len(), 1, "{plan:#}");
3568 assert_eq!(selected[0]["run"], clean.id);
3569 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3570 .await
3571 .unwrap();
3572 assert!(!clean.candidates[0].worktree.exists());
3573 assert!(dirty.candidates[0].worktree.exists());
3574 assert!(
3575 RunState::load_under(&dirty.id, &f.home)
3576 .unwrap()
3577 .retired_to
3578 .is_none()
3579 );
3580 }
3581
3582 #[tokio::test]
3583 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3584 let f = Fx::new().await;
3585 let mut source = f.run("aa00", true).await;
3586 let alternative = f.run("aa01", true).await;
3587 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3588 .await
3589 .unwrap();
3590 source.reviewed_commits = Some(vec![head.clone()]);
3591 source.save_under(&f.home).unwrap();
3592 let mut runs = vec![source.clone(), alternative.clone()];
3593 for i in 2..14 {
3594 runs.push(f.run(&format!("aa{i:02}"), false).await);
3595 }
3596 let mut task = f.task(&runs);
3597 task.pr_recovery = Some(crate::pr_recovery::Selection {
3598 url: "https://example.invalid/pull/1".into(),
3599 branch: "selected".into(),
3600 head: head.clone(),
3601 source_head: Some(head.clone()),
3602 source_run: Some(source.id.clone()),
3603 source_proof: None,
3604 disposition: Some(crate::pr_recovery::Disposition::Land),
3605 instruction: task.instruction.clone(),
3606 approved_at: Timestamp::now(),
3607 });
3608 source.pr_recovery = task.pr_recovery.clone();
3609 source.save_under(&f.home).unwrap();
3610 runs[0] = source.clone();
3611 crate::queue::Queue::at(f.home.join("queue"))
3612 .put(&mut task)
3613 .unwrap();
3614 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3615 let task_before = serde_json::to_value(&task).unwrap();
3616 let mut report = Report::default();
3617 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3618 assert_eq!(
3619 report.retired, 0,
3620 "automatic cleanup must still refuse blanket markers"
3621 );
3622 assert!(
3623 report
3624 .retained
3625 .iter()
3626 .any(|reason| reason.contains("explicitly retained"))
3627 );
3628 let mut total = 0;
3629 for expected in [5, 5, 2, 0] {
3630 let plan = reconciliation_plan(&f, &task, &source).await;
3631 let selected: Vec<_> = plan["candidates"]
3632 .as_array()
3633 .unwrap()
3634 .iter()
3635 .filter(|e| e["selected"] == true)
3636 .collect();
3637 assert_eq!(selected.len(), expected, "{plan:#}");
3638 for e in &selected {
3639 assert!(
3640 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3641 .unwrap()
3642 .retired_to
3643 .is_none()
3644 );
3645 }
3646 let operation = plan["id"].as_str().unwrap();
3647 for _ in 0..2 {
3648 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3649 .await
3650 .unwrap();
3651 }
3652 total += expected;
3653 }
3654 assert_eq!(total, 12);
3655 for run in &runs {
3656 let dir = f.home.join("runs").join(&run.id);
3657 assert!(dir.join("run.json").exists());
3658 assert!(dir.join("retain-recovery-sources").exists());
3659 assert!(
3660 git::branch_exists(&f.repo, &run.candidates[0].branch)
3661 .await
3662 .unwrap()
3663 );
3664 }
3665 assert!(source.candidates[0].worktree.exists());
3666 assert!(alternative.candidates[0].worktree.exists());
3667 assert_eq!(
3668 git::rev_parse(&f.repo, &source.candidates[0].branch)
3669 .await
3670 .unwrap(),
3671 head
3672 );
3673 assert_eq!(
3674 serde_json::to_value(
3675 crate::queue::Queue::at(f.home.join("queue"))
3676 .get(&task.id)
3677 .unwrap()
3678 )
3679 .unwrap(),
3680 task_before
3681 );
3682 assert_eq!(
3683 runs.iter()
3684 .filter(|r| RunState::load_under(&r.id, &f.home)
3685 .unwrap()
3686 .retired_to
3687 .is_none())
3688 .count(),
3689 2
3690 );
3691 }
3692
3693 #[tokio::test]
3694 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3695 let f = Fx::new().await;
3696 let source = f.run("ab00", true).await;
3697 let mut runs = vec![source.clone()];
3698 for i in 1..9 {
3699 runs.push(f.run(&format!("ab{i:02}"), false).await);
3700 }
3701 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3702 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3703 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3704 std::fs::write(
3705 runs[3].candidates[0].worktree.join("scratch/source"),
3706 "ignored source",
3707 )
3708 .unwrap();
3709 std::fs::write(
3710 runs[4].worktree_root().join("unknown"),
3711 "unregistered source",
3712 )
3713 .unwrap();
3714 let checkpoint = git::git(
3715 &runs[5].candidates[0].worktree,
3716 &[
3717 "rev-parse",
3718 "--path-format=absolute",
3719 "--git-path",
3720 "sequencer",
3721 ],
3722 )
3723 .await
3724 .unwrap();
3725 std::fs::create_dir(checkpoint.trim()).unwrap();
3726 runs[6].driver_exited = false;
3727 runs[6].driver_pid = Some(std::process::id());
3728 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3729 runs[6].save_under(&f.home).unwrap();
3730 runs[7].driver_exited = false;
3731 runs[7].driver_pid = None;
3732 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3734 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3735
3736 let hidden = &runs[8].candidates[0].worktree;
3737 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3738 .await
3739 .unwrap();
3740 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3741 assert!(
3742 git::git(hidden, &["status", "--porcelain"])
3743 .await
3744 .unwrap()
3745 .is_empty()
3746 );
3747 let task = f.task(&runs);
3748 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3749 let plan = reconciliation_plan(&f, &task, &source).await;
3750 assert!(
3751 plan["candidates"]
3752 .as_array()
3753 .unwrap()
3754 .iter()
3755 .all(|e| e["selected"] == false),
3756 "{plan:#}"
3757 );
3758 for run in &runs {
3759 assert!(run.candidates[0].worktree.exists());
3760 }
3761 }
3762
3763 #[tokio::test]
3764 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3765 let f = Fx::new().await;
3766 let source = f.run("ac00", true).await;
3767 let empty = f.run("ac01", false).await;
3768 let task = f.task(&[source.clone(), empty.clone()]);
3769 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3770 let plan = reconciliation_plan(&f, &task, &source).await;
3771 std::fs::write(
3772 f.home.join("runs").join(&source.id).join("new-report"),
3773 "new review checkpoint",
3774 )
3775 .unwrap();
3776 assert!(
3777 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3778 .await
3779 .is_err()
3780 );
3781 assert!(empty.candidates[0].worktree.exists());
3782 assert!(
3783 RunState::load_under(&empty.id, &f.home)
3784 .unwrap()
3785 .retired_to
3786 .is_none()
3787 );
3788 let marker = f
3789 .home
3790 .join("runs")
3791 .join(&empty.id)
3792 .join("retain-recovery-sources");
3793 std::fs::write(&marker, "different owner").unwrap();
3794 let plan = reconciliation_plan(&f, &task, &source).await;
3795 assert!(
3796 plan["candidates"]
3797 .as_array()
3798 .unwrap()
3799 .iter()
3800 .all(|e| e["selected"] == false)
3801 );
3802 }
3803
3804 #[tokio::test]
3805 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3806 for partial in [false, true] {
3807 let f = Fx::new().await;
3808 let source = f.run("ab10", true).await;
3809 let first = f.run("ab11", false).await;
3810 let second = f.run("ab12", false).await;
3811 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3812 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3813 let plan = reconciliation_plan(&f, &task, &source).await;
3814 let id = plan["id"].as_str().unwrap();
3815 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3816 std::fs::write(&staging, b"").unwrap();
3818 assert!(
3819 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3820 .await
3821 .is_err()
3822 );
3823 assert!(staging.exists());
3824 std::fs::remove_file(&staging).unwrap();
3825 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3826 .await
3827 .unwrap_err();
3828 assert!(
3829 error.to_string().contains("before run state rename"),
3830 "{error:#}"
3831 );
3832 assert!(
3833 RunState::load_under(&first.id, &f.home)
3834 .unwrap()
3835 .retired_to
3836 .is_some()
3837 );
3838 assert!(
3839 RunState::load_under(&second.id, &f.home)
3840 .unwrap()
3841 .retired_to
3842 .is_none()
3843 );
3844 assert!(first.candidates[0].worktree.exists());
3845 assert!(second.candidates[0].worktree.exists());
3846 let interrupted = std::fs::read(&staging).unwrap();
3847 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3849 assert!(
3850 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3851 .await
3852 .is_err()
3853 );
3854 assert_eq!(
3855 std::fs::read(&staging).unwrap(),
3856 b"unrelated recovery source"
3857 );
3858 std::fs::write(&staging, interrupted).unwrap();
3859 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3860 .await
3861 .unwrap();
3862 assert!(!staging.exists());
3863 assert!(!first.candidates[0].worktree.exists());
3864 assert!(!second.candidates[0].worktree.exists());
3865 assert_eq!(
3866 RunState::load_under(&second.id, &f.home)
3867 .unwrap()
3868 .retired_to,
3869 Some(source.id)
3870 );
3871 }
3872 }
3873
3874 #[tokio::test]
3875 async fn reconciliation_resumes_only_its_exact_durable_fence() {
3876 let f = Fx::new().await;
3877 let source = f.run("ad00", true).await;
3878 let empty = f.run("ad01", false).await;
3879 let task = f.task(&[source.clone(), empty.clone()]);
3880 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3881 let plan = reconciliation_plan(&f, &task, &source).await;
3882 let id = plan["id"].as_str().unwrap();
3883 assert!(
3884 reconcile::interrupt_after_fence(&f.home, &task, id)
3885 .await
3886 .is_err()
3887 );
3888 assert!(empty.candidates[0].worktree.exists());
3889 assert_eq!(
3890 RunState::load_under(&empty.id, &f.home)
3891 .unwrap()
3892 .retired_to
3893 .as_ref(),
3894 Some(&source.id)
3895 );
3896 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3897 .await
3898 .unwrap();
3899 assert!(!empty.candidates[0].worktree.exists());
3900 git::worktree_add_branch(
3902 &f.repo,
3903 &empty.candidates[0].worktree,
3904 "restore-empty",
3905 &f.base,
3906 )
3907 .await
3908 .unwrap();
3909 assert!(
3911 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3912 .await
3913 .is_err()
3914 );
3915 assert!(empty.candidates[0].worktree.exists());
3916 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3917 changed.prep_failure = Some("new owner edit".into());
3918 changed.save_under(&f.home).unwrap();
3919 assert!(
3920 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3921 .await
3922 .is_err()
3923 );
3924 }
3925
3926 #[tokio::test]
3927 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3928 let f = Fx::new().await;
3929 let source = f.run("ae00", true).await;
3930 let mut prep = f.run("ae01", false).await;
3931 let candidate = prep.candidates.pop().unwrap();
3932 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3933 prep.save_under(&f.home).unwrap();
3934 let reset = f.run("ae02", false).await;
3935 let wt = &reset.candidates[0].worktree;
3936 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3937 git::git(wt, &["add", "."]).await.unwrap();
3938 git::git(wt, &["commit", "-m", "unique implementation"])
3939 .await
3940 .unwrap();
3941 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3942 let unknown = f.run("ae03", false).await;
3943 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3944 let mut raw: serde_json::Value =
3945 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3946 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3947 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3948 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3949 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3950 let plan = reconciliation_plan(&f, &task, &source).await;
3951 let selected: Vec<_> = plan["candidates"]
3952 .as_array()
3953 .unwrap()
3954 .iter()
3955 .filter(|e| e["selected"] == true)
3956 .collect();
3957 assert_eq!(selected.len(), 1, "{plan:#}");
3958 assert_eq!(selected[0]["run"], prep.id);
3959 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3960 .await
3961 .unwrap();
3962 assert!(!candidate.worktree.exists());
3963 assert!(reset.candidates[0].worktree.exists());
3964 assert!(unknown.candidates[0].worktree.exists());
3965 }
3966
3967 #[tokio::test]
3968 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3969 let f = Fx::new().await;
3970 let mut state = f.run("aaaa", true).await;
3971 let candidate = state.candidates[0].worktree.clone();
3972 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3973 Fx::target(&candidate);
3974 for seat in ["review-1", "review-2"] {
3975 let wt = state.worktree_root().join(seat);
3976 git::worktree_add_detached(&f.repo, &wt, &f.base)
3977 .await
3978 .unwrap();
3979 Fx::target(&wt);
3980 }
3981 state.status = RunStatus::Landing;
3982 state.save_under(&f.home).unwrap();
3983 let second = Fx::new().await;
3984 let other = second.run("bbbb", true).await;
3985 let mut imported = other.clone();
3987 imported.save_under(&f.home).unwrap();
3988 Fx::target(&other.candidates[0].worktree);
3989 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3990 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3991 assert!(
3992 candidate.join("target/debug/output").exists(),
3993 "active writer protected"
3994 );
3995 assert!(
3996 !state
3997 .worktree_root()
3998 .join("review-1/target/debug/output")
3999 .exists()
4000 );
4001 assert!(
4002 !state
4003 .worktree_root()
4004 .join("review-2/target/debug/output")
4005 .exists()
4006 );
4007 assert!(
4008 !other.candidates[0]
4009 .worktree
4010 .join("target/debug/output")
4011 .exists()
4012 );
4013 assert!(out.freed_bytes >= 600);
4014 drop(guard);
4015 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4016 assert!(out.freed_bytes >= 200);
4017 assert_eq!(
4018 std::fs::read_to_string(candidate.join("source")).unwrap(),
4019 "uncommitted work"
4020 );
4021 assert!(
4022 git::branch_exists(&f.repo, &state.candidates[0].branch)
4023 .await
4024 .unwrap()
4025 );
4026 }
4027
4028 #[tokio::test]
4029 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
4030 let f = Fx::new().await;
4031 let state = f.run("cc01", false).await;
4032 let wt = &state.candidates[0].worktree;
4033 Fx::target(wt);
4034 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
4035 let dir = wt.join("target").join(dir);
4036 std::fs::create_dir_all(&dir).unwrap();
4037 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
4038 }
4039 let manifest = wt.join("target/debug/output.d");
4041 let output = wt.join("target/debug/output");
4042 std::fs::write(
4043 &manifest,
4044 format!(
4045 "{}: {}\n",
4046 output.display(),
4047 wt.join("target/debug/notes.txt").display()
4048 ),
4049 )
4050 .unwrap();
4051 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4052 assert_eq!(report.freed_bytes, 200);
4053 assert!(!output.exists());
4054 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
4055 assert_eq!(
4056 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
4057 "user evidence"
4058 );
4059 }
4060 assert!(manifest.exists());
4062 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
4063 std::fs::write(&output, "user note replacing an old binary").unwrap();
4064 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4065 assert_eq!(report.freed_bytes, 0);
4066 assert_eq!(
4067 std::fs::read_to_string(&output).unwrap(),
4068 "user note replacing an old binary"
4069 );
4070 }
4071
4072 #[tokio::test]
4073 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
4074 let f = Fx::new().await;
4075 let state = f.run("cc02", false).await;
4076 let wt = &state.candidates[0].worktree;
4077 Fx::target(wt);
4078 let mut disk = Fx::disk();
4079 disk.worktree_limit_bytes = u64::MAX;
4080 let now = Fx::now();
4081 let first = sweep(&f.home, &disk, now).await;
4082 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
4083 assert_eq!(
4084 sweep(&f.home, &disk, now).await.measured_bytes,
4085 first.measured_bytes
4086 );
4087 assert!(
4088 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
4089 .await
4090 .measured_bytes
4091 > first.measured_bytes
4092 );
4093 disk.worktree_limit_bytes = 0;
4094 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
4095 }
4096
4097 #[tokio::test]
4098 async fn unfinished_source_fold_reports_its_retention_reason() {
4099 let f = Fx::new().await;
4100 let mut source = f.run("cc03", true).await;
4101 let task = f.task(&[source.clone()]);
4102 let error = crate::graph::fold_run(&mut source, true, &f.home)
4103 .await
4104 .unwrap_err();
4105 assert!(error.to_string().contains(&task.id));
4106 assert!(source.candidates[0].worktree.exists());
4107 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4108 let error = crate::graph::fold_run(&mut source, true, &f.home)
4109 .await
4110 .unwrap_err();
4111 assert!(error.to_string().contains("explicitly retained"));
4112 }
4113
4114 #[tokio::test]
4115 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
4116 let f = Fx::new().await;
4117 let mut source = f.run("cc04", true).await;
4118 let mut task = f.task(&[source.clone()]);
4119 task.hold_manual(Some("queue settlement pending".into()));
4120 let queue = crate::queue::Queue::at(f.home.join("queue"));
4121 queue.put(&mut task).unwrap();
4122 source.merge = Some(crate::run::MergeOutcome {
4123 mode: crate::config::MergeMode::Pr,
4124 ok: true,
4125 detail: "confirmed merged".into(),
4126 empty: false,
4127 });
4128 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
4130 source.status = status;
4131 assert!(
4132 crate::graph::fold_run(&mut source, true, &f.home)
4133 .await
4134 .is_err()
4135 );
4136 assert!(source.candidates[0].worktree.exists());
4137 }
4138 git::git(
4139 &f.repo,
4140 &["merge", "--ff-only", &source.candidates[0].branch],
4141 )
4142 .await
4143 .unwrap();
4144 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
4145 .await
4146 .unwrap();
4147 source.status = RunStatus::Merged;
4148 source.save_under(&f.home).unwrap();
4149 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
4150 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4151 assert!(
4152 crate::graph::fold_run(&mut source, true, &f.home)
4153 .await
4154 .unwrap_err()
4155 .to_string()
4156 .contains("explicitly retained")
4157 );
4158 std::fs::remove_file(
4161 f.home
4162 .join("runs")
4163 .join(&source.id)
4164 .join("retain-recovery-sources"),
4165 )
4166 .unwrap();
4167 let (folded, unreadable) = crate::clean::fold_due(
4168 &f.home.join("runs"),
4169 &f.home,
4170 &f.bay,
4171 &Fx::disk(),
4172 Fx::now(),
4173 )
4174 .await
4175 .unwrap();
4176 assert_eq!((folded, unreadable), (1, 0));
4177 assert!(!source.candidates[0].worktree.exists());
4178 let evidence = f
4179 .home
4180 .join("runs")
4181 .join(&source.id)
4182 .join("artifacts/fold-evidence");
4183 let names: Vec<_> = std::fs::read_dir(evidence)
4184 .unwrap()
4185 .map(|entry| entry.unwrap().file_name())
4186 .collect();
4187 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
4188 assert_eq!(
4189 queue.get(&task.id).unwrap().status,
4190 crate::queue::TaskStatus::Held
4191 );
4192 }
4193
4194 #[tokio::test]
4195 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
4196 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
4197 let f = Fx::new().await;
4198 let mut state = f.run("cc06", true).await;
4199 let candidate = state.candidates[0].clone();
4200 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4201 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
4202 git::rescue_commit(&candidate.worktree, "second candidate change")
4203 .await
4204 .unwrap();
4205 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4206 state.tally = Some(crate::run::Tally {
4207 first_choice: Default::default(),
4208 borda: Default::default(),
4209 winner: 'A',
4210 rankings: 0,
4211 unanimous_initial: true,
4212 deliberated: false,
4213 changed_votes: 0,
4214 unanimous_final: true,
4215 tie_break: None,
4216 judges: 0,
4217 present: 0,
4218 quorum: 0,
4219 met_quorum: true,
4220 uncontested: Some("fixture".into()),
4221 });
4222 let reviewer = state.worktree_root().join("review-1");
4225 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
4226 .await
4227 .unwrap();
4228 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
4231 git::rescue_commit(&f.repo, "advance before squash")
4232 .await
4233 .unwrap();
4234 let published = if variant == "partial" {
4235 &intermediate
4236 } else {
4237 &head
4238 };
4239 git::git(&f.repo, &["merge", "--squash", published])
4240 .await
4241 .unwrap();
4242 git::git(&f.repo, &["commit", "-m", "squash candidate"])
4243 .await
4244 .unwrap();
4245 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
4246 assert!(
4247 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
4248 .await
4249 .unwrap()
4250 .ok()
4251 );
4252 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
4253 git::rescue_commit(&f.repo, "advance after squash")
4254 .await
4255 .unwrap();
4256 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
4257 .await
4258 .unwrap();
4259 if matches!(variant, "checkpoint" | "reviewer") {
4260 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
4262 git::rescue_commit(&reviewer, "unpublished checkpoint")
4263 .await
4264 .unwrap();
4265 if variant == "checkpoint" {
4266 git::git(&reviewer, &["reset", "--hard", &intermediate])
4267 .await
4268 .unwrap();
4269 }
4270 }
4271 if variant == "loser" {
4272 let mut loser = candidate.clone();
4273 loser.label = 'B';
4274 loser.branch = "magi/cc06/B".into();
4275 loser.worktree = state.worktree_root().join("cand-B");
4276 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
4277 .await
4278 .unwrap();
4279 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
4280 git::rescue_commit(&loser.worktree, "independent loser")
4281 .await
4282 .unwrap();
4283 state.candidates.push(loser);
4284 }
4285 state.status = RunStatus::Merged;
4286 state.merge = Some(crate::run::MergeOutcome {
4287 mode: crate::config::MergeMode::Pr,
4288 ok: true,
4289 detail: "candidate A squash merged".into(),
4290 empty: false,
4291 });
4292 state.save_under(&f.home).unwrap();
4293 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
4294 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
4295 result.unwrap();
4296 assert!(
4297 !candidate.worktree.exists() && !reviewer.exists(),
4298 "{variant}"
4299 );
4300 assert!(
4301 !git::branch_exists(&f.repo, &candidate.branch)
4302 .await
4303 .unwrap()
4304 );
4305 let artifacts = f
4306 .home
4307 .join("runs")
4308 .join(&state.id)
4309 .join("artifacts/fold-evidence");
4310 assert_eq!(
4311 artifacts.join(format!("{head}.patch")).exists(),
4312 variant == "partial",
4313 "only a partially published winner needs evidence: {variant}"
4314 );
4315 assert_eq!(
4316 artifacts.join(format!("{intermediate}.patch")).exists(),
4317 variant == "partial",
4318 "squash proof also covers intermediate reviewer tips: {variant}"
4319 );
4320 let patches: Vec<_> = std::fs::read_dir(&artifacts)
4321 .unwrap()
4322 .map(|entry| entry.unwrap().path())
4323 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
4324 .collect();
4325 let expected = match variant {
4326 "partial" => 2,
4327 "loser" | "reviewer" => 1,
4328 _ => 0,
4329 };
4330 assert_eq!(patches.len(), expected, "{variant}");
4331 if variant == "loser" {
4332 assert!(
4333 std::fs::read_to_string(&patches[0])
4334 .unwrap()
4335 .contains("losing candidate")
4336 );
4337 }
4338 assert!(artifacts.join("manifest.json").exists());
4339 if variant == "reviewer" {
4340 assert!(
4341 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
4342 .unwrap()
4343 .contains("reviewer checkpoint")
4344 );
4345 }
4346 crate::graph::fold_run(&mut state, true, &f.home)
4347 .await
4348 .unwrap();
4349 }
4350 }
4351
4352 #[tokio::test]
4353 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
4354 for missing in [true, false] {
4355 let f = Fx::new().await;
4356 let mut state = f.run("cc05", true).await;
4357 git::git(
4360 &f.repo,
4361 &["update-ref", "refs/remotes/origin/main", &f.base],
4362 )
4363 .await
4364 .unwrap();
4365 let candidate = state.candidates[0].clone();
4366 if !missing {
4367 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
4368 git::rescue_commit(&f.repo, "advance base before rebase")
4369 .await
4370 .unwrap();
4371 git::git(&candidate.worktree, &["rebase", "main"])
4372 .await
4373 .unwrap();
4374 git::sync_to_head(&candidate.worktree).await.unwrap();
4377 }
4378 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4379 state.reviewed_commits = Some(Vec::new());
4381 state.candidates[0].worktree = state.worktree_root().join("under-review");
4382 git::git(
4383 &f.repo,
4384 &[
4385 "worktree",
4386 "move",
4387 candidate.worktree.to_str().unwrap(),
4388 state.candidates[0].worktree.to_str().unwrap(),
4389 ],
4390 )
4391 .await
4392 .unwrap();
4393 let reviewer = state.worktree_root().join("review-1");
4394 git::worktree_add_detached(&f.repo, &reviewer, &head)
4395 .await
4396 .unwrap();
4397 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
4400 git::rescue_commit(&reviewer, "old review history")
4401 .await
4402 .unwrap();
4403 git::git(&reviewer, &["reset", "--hard", &head])
4404 .await
4405 .unwrap();
4406 if missing {
4407 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
4408 }
4409 let run_dir = f.home.join("runs").join(&state.id);
4410 let artifacts = run_dir.join("artifacts");
4411 std::fs::create_dir_all(&artifacts).unwrap();
4412 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
4413 if missing {
4414 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
4415 }
4416 state.status = RunStatus::Merged;
4417 state.merge = Some(crate::run::MergeOutcome {
4418 mode: crate::config::MergeMode::Pr,
4419 ok: true,
4420 detail: "confirmed squash merge".into(),
4421 empty: false,
4422 });
4423 state.save_under(&f.home).unwrap();
4424 let owned = state.candidates[0].worktree.clone();
4426 state.candidates[0].worktree = f.home.join("missing-external");
4427 assert!(
4428 crate::graph::fold_run(&mut state, true, &f.home)
4429 .await
4430 .unwrap_err()
4431 .to_string()
4432 .contains("outside the inventoried")
4433 );
4434 state.candidates[0].worktree = owned;
4435 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
4437 assert!(
4438 crate::graph::fold_run(&mut state, true, &f.home)
4439 .await
4440 .is_err()
4441 );
4442 assert!(reviewer.exists());
4443 drop(guard);
4444 let evidence = artifacts.join("fold-evidence");
4446 std::fs::write(&evidence, "blocked").unwrap();
4447 assert!(
4448 crate::graph::fold_run(&mut state, true, &f.home)
4449 .await
4450 .is_err()
4451 );
4452 assert!(reviewer.exists());
4453 std::fs::remove_file(&evidence).unwrap();
4454 std::fs::create_dir_all(&evidence).unwrap();
4455 let oversized = evidence.join("capacity-fixture");
4456 std::fs::File::create(&oversized)
4457 .unwrap()
4458 .set_len(64 * 1024 * 1024)
4459 .unwrap();
4460 assert!(
4461 crate::graph::fold_run(&mut state, true, &f.home)
4462 .await
4463 .is_err()
4464 );
4465 assert!(reviewer.exists());
4466 std::fs::remove_file(oversized).unwrap();
4467 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
4470 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
4471 assert!(
4472 crate::graph::fold_run(&mut state, true, &f.home)
4473 .await
4474 .is_err()
4475 );
4476 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
4477 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
4478 .await
4479 .unwrap();
4480 std::fs::write(&checkpoint, &head).unwrap();
4481 assert!(
4482 crate::graph::fold_run(&mut state, true, &f.home)
4483 .await
4484 .is_err()
4485 );
4486 std::fs::remove_file(checkpoint).unwrap();
4487 let (folded, unreadable) = crate::clean::fold_due(
4488 &f.home.join("runs"),
4489 &f.home,
4490 &f.bay,
4491 &Fx::disk(),
4492 Fx::now(),
4493 )
4494 .await
4495 .unwrap();
4496 if folded != 1 {
4497 panic!(
4498 "normal cleanup failed: {:#}",
4499 crate::graph::fold_run(&mut state, true, &f.home)
4500 .await
4501 .unwrap_err()
4502 );
4503 }
4504 assert_eq!((folded, unreadable), (1, 0));
4505 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4506 state = RunState::load_under(&state.id, &f.home).unwrap();
4507 assert!(state.candidates[0].folded);
4508 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4509 crate::graph::fold_run(&mut state, true, &f.home)
4510 .await
4511 .unwrap();
4512 assert_eq!(
4513 std::fs::read(evidence.join("manifest.json")).unwrap(),
4514 manifest
4515 );
4516 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4517 .await
4518 .unwrap();
4519 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4520 assert!(
4521 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4522 .await
4523 .unwrap()
4524 .ok()
4525 );
4526 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4529 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4530 .as_str()
4531 .unwrap();
4532 let recovered = f.home.join("recovered");
4533 std::fs::create_dir_all(&recovered).unwrap();
4534 git::git(&recovered, &["init"]).await.unwrap();
4535 std::fs::write(
4536 recovered.join("source"),
4537 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4538 )
4539 .unwrap();
4540 git::git(
4541 &recovered,
4542 &[
4543 "apply",
4544 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4545 ],
4546 )
4547 .await
4548 .unwrap();
4549 assert_eq!(
4550 std::fs::read_to_string(recovered.join("source")).unwrap(),
4551 "cc05"
4552 );
4553 assert!(
4554 !std::fs::read(evidence.join(format!("{head}.patch")))
4555 .unwrap()
4556 .is_empty()
4557 );
4558 assert_eq!(
4559 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4560 "review findings"
4561 );
4562 if missing {
4563 assert_eq!(
4564 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4565 "original snapshot"
4566 );
4567 } else {
4568 assert!(!artifacts.join("cand-A.patch").exists());
4569 }
4570 assert!(run_dir.join("run.json").exists());
4571 }
4572 }
4573
4574 #[tokio::test]
4575 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4576 use std::os::unix::fs::PermissionsExt as _;
4577 let f = Fx::new().await;
4578 let source = f.run("dd03", true).await;
4579 let empty = f.run("dd04", false).await;
4580 let task = f.task(&[source, empty.clone()]);
4581 let entered = f.repo.join("proof-entered");
4582 let release = f.repo.join("proof-release");
4583 let hook = f.repo.join("proof-monitor");
4584 std::fs::write(&hook, format!(
4585 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4586 entered.display(), release.display())).unwrap();
4587 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4588 git::git(
4589 &f.repo,
4590 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4591 )
4592 .await
4593 .unwrap();
4594 struct ReleaseProof(PathBuf);
4595 impl Drop for ReleaseProof {
4596 fn drop(&mut self) {
4597 let _ = std::fs::write(&self.0, "continue");
4598 }
4599 }
4600 let _release_on_panic = ReleaseProof(release.clone());
4601 let home = f.home.clone();
4602 let sweep = tokio::spawn(async move {
4603 let mut report = Report::default();
4604 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4605 report
4606 });
4607 let watchdog = std::time::Duration::from_secs(60);
4610 tokio::time::timeout(watchdog, async {
4611 while !entered.exists() {
4612 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4613 }
4614 })
4615 .await
4616 .expect("Git retirement proof did not enter the blocking hook");
4617 let queue = crate::queue::Queue::at(f.home.join("queue"));
4618 let mut selected = queue.get(&task.id).unwrap();
4619 selected.resume_override = Some(crate::queue::OperatorResume {
4620 question_id: "during-slow-proof".into(),
4621 at: Fx::now(),
4622 conductor_rehold: None,
4623 forced: false,
4624 pinned_run: Some(empty.id.clone()),
4625 });
4626 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4627 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4631 Ok(result) => {
4632 result.unwrap().unwrap();
4633 true
4634 }
4635 Err(_) => {
4636 std::fs::write(&release, "continue").unwrap();
4637 update.await.unwrap().unwrap();
4638 false
4639 }
4640 };
4641 std::fs::write(release, "continue").unwrap();
4642 let report = tokio::time::timeout(watchdog, sweep)
4643 .await
4644 .expect("Git retirement proof did not finish after release")
4645 .unwrap();
4646 assert!(
4647 updated_while_blocked,
4648 "task writers must not wait behind Git retirement proof"
4649 );
4650 assert_eq!(report.retired, 0);
4651 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4652 assert!(empty.candidates[0].worktree.exists());
4653 assert!(
4654 RunState::load_under(&empty.id, &f.home)
4655 .unwrap()
4656 .retired_to
4657 .is_none()
4658 );
4659 }
4660
4661 #[tokio::test]
4662 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4663 let f = Fx::new().await;
4664 let source = f.run("dd01", true).await;
4665 let mut empty = f.run("dd02", false).await;
4666 let expected = f.task(&[source.clone(), empty.clone()]);
4667 let queue = crate::queue::Queue::at(f.home.join("queue"));
4668 let _claim = queue.claim(&expected.id).unwrap();
4669 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4671 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4672 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4673 let mut selected = queue.get(&expected.id).unwrap();
4674 selected.resume_override = Some(crate::queue::OperatorResume {
4675 question_id: "new-owner-decision".into(),
4676 at: Fx::now(),
4677 conductor_rehold: None,
4678 forced: false,
4679 pinned_run: Some(empty.id.clone()),
4680 });
4681 queue.put(&mut selected).unwrap();
4682 let error = commit_retirement(
4683 &f.home,
4684 &queue,
4685 &expected,
4686 &mut empty,
4687 &source.id,
4688 &expected.runs,
4689 Fx::now(),
4690 )
4691 .unwrap_err();
4692 assert!(error.to_string().contains("task changed"));
4693 assert!(
4694 RunState::load_under(&empty.id, &f.home)
4695 .unwrap()
4696 .retired_to
4697 .is_none()
4698 );
4699 assert!(empty.candidates[0].worktree.exists());
4700 assert_eq!(
4701 queue
4702 .get(&expected.id)
4703 .unwrap()
4704 .resume_override
4705 .unwrap()
4706 .pinned_run,
4707 Some(empty.id.clone())
4708 );
4709 assert!(
4710 !f.home
4711 .join("recovery-lineages")
4712 .join(format!("{}.json", expected.id))
4713 .exists()
4714 );
4715 assert!(
4717 commit_retirement(
4718 &f.home,
4719 &queue,
4720 &selected,
4721 &mut empty,
4722 &source.id,
4723 &selected.runs,
4724 Fx::now()
4725 )
4726 .unwrap_err()
4727 .to_string()
4728 .contains("selected this recovery source")
4729 );
4730 }
4731
4732 #[tokio::test]
4733 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4734 let f = Fx::new().await;
4735 let state = f.run("cccc", false).await;
4736 let wt = &state.candidates[0].worktree;
4737 Fx::target(wt);
4738 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4739 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4740 assert!(
4741 !declared_outputs(&wt.join("target"))
4742 .unwrap()
4743 .contains(&wt.join("target/debug/link"))
4744 );
4745 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4746 let cargo = cargo_locks(&wt.join("target")).unwrap();
4747 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4748 assert_eq!(out.freed_bytes, 0);
4749 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4750 drop(cargo);
4751 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4752 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4753 assert_eq!(out.freed_bytes, 0);
4754 drop(driver);
4755 let exclusive = OutputLease::take(wt, true).unwrap();
4756 assert!(
4757 OutputLease::borrow(wt).is_err(),
4758 "resume cannot start a writer during cleanup"
4759 );
4760 drop(exclusive);
4761 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4762 assert_eq!(
4763 std::fs::read(wt.join("target/notes")).unwrap(),
4764 b"user file"
4765 );
4766 }
4767
4768 #[tokio::test]
4769 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4770 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4771 let f = Fx::new().await;
4772 let source = f.run("aa10", true).await;
4773 let empty = f.run("aa11", false).await;
4774 f.task(&[source, empty.clone()]);
4775 let wt = &empty.candidates[0].worktree;
4776 Fx::target(wt);
4777 let path = wt.join("target").join(leftover);
4778 if leftover.ends_with(".log") {
4779 std::fs::write(&path, "keep log\n").unwrap();
4780 } else {
4781 std::fs::create_dir(&path).unwrap();
4783 }
4784 let mut disk = Fx::disk();
4785 disk.auto_fold = true;
4786 let report = sweep(&f.home, &disk, Fx::now()).await;
4787 assert_eq!(report.freed_bytes, 200, "{report:?}");
4788 assert_eq!(report.retired, 0, "{report:?}");
4789 assert!(!wt.join("target/debug/output").exists());
4790 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4791 assert!(path.exists());
4792 if leftover.ends_with(".log") {
4793 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4794 }
4795 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4796 assert!(worktrees(&empty).await.unwrap().contains(wt));
4797 assert!(
4798 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4799 .await
4800 .unwrap()
4801 );
4802 assert!(
4803 RunState::load_under(&empty.id, &f.home)
4804 .unwrap()
4805 .retired_to
4806 .is_none()
4807 );
4808 }
4809 }
4810
4811 #[tokio::test]
4812 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4813 let f = Fx::new().await;
4814 let state = f.run("aa12", false).await;
4815 let wt = &state.candidates[0].worktree;
4816 Fx::target(wt);
4817 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4818 assert_eq!(report.freed_bytes, 200, "{report:?}");
4819 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4820 for name in ["tmp", "magi-tools", "log"] {
4821 let path = wt.join("target").join(name);
4822 std::fs::create_dir(&path).unwrap();
4823 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4824 std::fs::remove_dir(path).unwrap();
4825 }
4826 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4827 }
4828
4829 #[tokio::test]
4830 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4831 for invalid in [
4832 "tag",
4833 "identity",
4834 "tracked",
4835 "profile-file",
4836 "profile-symlink",
4837 "identity-symlink",
4838 ] {
4839 let f = Fx::new().await;
4840 let state = f.run("aa13", false).await;
4841 let wt = &state.candidates[0].worktree;
4842 Fx::target(wt);
4843 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4844 let target = wt.join("target");
4845 match invalid {
4846 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4847 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4848 "tracked" => {
4849 git::git(wt, &["add", "-f", "target/debug/output"])
4850 .await
4851 .unwrap();
4852 }
4853 "profile-file" => {
4854 std::fs::write(target.join("release"), "keep").unwrap();
4855 }
4856 "profile-symlink" => {
4857 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4858 .unwrap();
4859 }
4860 "identity-symlink" => {
4861 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4862 .unwrap();
4863 std::os::unix::fs::symlink(
4864 target.join("saved-info"),
4865 target.join(".rustc_info.json"),
4866 )
4867 .unwrap();
4868 }
4869 _ => unreachable!(),
4870 }
4871 let before = std::fs::read(target.join("debug/output")).unwrap();
4872 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4873 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4874 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4875 assert!(target.join("tmp").is_dir());
4876 }
4877 }
4878
4879 #[tokio::test]
4880 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4881 let f = Fx::new().await;
4882 let source = f.run("0000", true).await;
4883 let empty = f.run("0001", false).await;
4884 let changed = f.run("0002", false).await;
4885 let unknown = f.run("0003", false).await;
4886 let unproven = f.run("0004", false).await;
4887 f.task(&[
4888 source.clone(),
4889 empty.clone(),
4890 changed.clone(),
4891 unknown.clone(),
4892 unproven.clone(),
4893 ]);
4894 for run in [&empty, &changed, &unknown] {
4895 Fx::target(&run.candidates[0].worktree);
4896 }
4897 let reviewer = empty.worktree_root().join("review-1");
4898 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4899 .await
4900 .unwrap();
4901 Fx::target(&reviewer);
4902 let mut disk = Fx::disk();
4903 disk.auto_fold = false;
4904 let report = sweep(&f.home, &disk, Fx::now()).await;
4905 assert_eq!(report.freed_bytes, 800, "{report:?}");
4906 let wt = &empty.candidates[0].worktree;
4907 assert!(wt.join("target/debug/output.d").exists());
4908 assert!(!wt.join("target/debug/output").exists());
4909 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4910 let proof: CargoResidue =
4911 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4912 assert_eq!(
4913 proof.files[Path::new("debug/output.d")],
4914 std::fs::read(wt.join("target/debug/output.d")).unwrap()
4915 );
4916 std::fs::write(
4917 changed.candidates[0].worktree.join("target/debug/output.d"),
4918 "owner notes",
4919 )
4920 .unwrap();
4921 std::fs::write(
4922 unknown.candidates[0].worktree.join("target/debug/notes"),
4923 "keep",
4924 )
4925 .unwrap();
4926 Fx::target(&unproven.candidates[0].worktree);
4927 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4928 disk.auto_fold = true;
4929 let locks = cargo_locks(&wt.join("target")).unwrap();
4930 let mut report = Report::default();
4931 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4932 assert_eq!(report.retired, 0, "{report:?}");
4933 assert!(
4934 report
4935 .retained
4936 .iter()
4937 .any(|reason| reason.contains("Cargo build"))
4938 );
4939 drop(locks);
4940 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4941 assert_eq!(report.retired, 1, "{report:?}");
4942 assert!(!wt.exists());
4943 assert!(!reviewer.exists());
4944 assert!(
4945 residue_proof(&f.home, &empty.id, &reviewer)
4946 .unwrap()
4947 .exists()
4948 );
4949 assert!(
4950 proof_path.exists(),
4951 "metadata evidence outlives the worktree"
4952 );
4953 assert_eq!(
4954 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4955 Some(source.id)
4956 );
4957 for run in [&changed, &unknown, &unproven] {
4958 assert!(run.candidates[0].worktree.exists());
4959 assert!(
4960 RunState::load_under(&run.id, &f.home)
4961 .unwrap()
4962 .retired_to
4963 .is_none()
4964 );
4965 }
4966 }
4967
4968 #[tokio::test]
4969 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4970 let f = Fx::new().await;
4971 let source = f.run("0000", true).await;
4972 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4973 .await
4974 .unwrap();
4975 let mut runs = vec![source.clone()];
4976 for n in 1..=12 {
4977 runs.push(f.run(&format!("{n:04}"), false).await);
4978 }
4979 let dirty = runs[1].candidates[0].worktree.clone();
4980 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4981 let ignored = runs[2].candidates[0].worktree.clone();
4982 std::fs::create_dir(ignored.join("scratch")).unwrap();
4983 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4984 runs[1].candidates.clear();
4987 runs[1].save_under(&f.home).unwrap();
4988 let task = f.task(&runs);
4989 let mut total = 0;
4990 for _ in 0..4 {
4991 let mut report = Report::default();
4992 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4993 assert!(report.retired <= 5);
4994 total += report.retired;
4995 }
4996 assert_eq!(total, 10);
4997 assert!(dirty.exists());
4998 assert!(ignored.exists());
4999 assert_eq!(
5000 git::rev_parse(&f.repo, &source.candidates[0].branch)
5001 .await
5002 .unwrap(),
5003 source_head
5004 );
5005 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
5006 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
5007 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
5008 let mut released = task.clone();
5009 released.release();
5010 assert_eq!(
5011 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
5012 RunState::load_under(id, &f.home)
5013 }),
5014 Some(source.id.clone())
5015 );
5016
5017 assert!(
5018 f.home
5019 .join("runs")
5020 .join(&retired.id)
5021 .join("run.json")
5022 .exists()
5023 );
5024 assert!(
5025 f.home
5026 .join("recovery-lineages")
5027 .join(format!("{}.json", task.id))
5028 .exists()
5029 );
5030 }
5031
5032 #[tokio::test]
5033 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
5034 let f = Fx::new().await;
5035 let a = f.run("dddd", true).await;
5036 let b = f.run("eeee", true).await;
5037 let empty = f.run("ffff", false).await;
5038 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
5039 assert!(
5040 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
5041 "unfinished queue history protects older source branches from ordinary folding"
5042 );
5043 let mut prior = a.clone();
5044 assert!(
5045 crate::graph::fold_run(&mut prior, true, &f.home)
5046 .await
5047 .unwrap_err()
5048 .to_string()
5049 .contains(&task.id)
5050 );
5051 let mut report = Report::default();
5052 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5053 assert_eq!(report.retired, 0);
5054 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
5055 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
5056 Fx::target(&a.candidates[0].worktree);
5057 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5058 assert!(
5059 report.freed_bytes > 0,
5060 "source retention does not retain regenerable outputs"
5061 );
5062 assert!(a.candidates[0].worktree.exists());
5063 assert!(b.candidates[0].worktree.exists());
5064 assert!(empty.candidates[0].worktree.exists());
5065 }
5066
5067 #[tokio::test]
5068 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
5069 let f = Fx::new().await;
5070 let authority = f.run("p000", true).await;
5071 let question_run = f.run("p001", false).await;
5072 let mut checkpoint = f.run("p002", false).await;
5073 checkpoint.land_approval = Some(crate::run::LandApproval {
5074 question: "pending".into(),
5075 head: f.base.clone(),
5076 });
5077 checkpoint.save_under(&f.home).unwrap();
5078 let retained = f.run("p003", false).await;
5079 std::fs::write(
5080 f.home
5081 .join("runs")
5082 .join(&retained.id)
5083 .join("retain-recovery-sources"),
5084 "owner",
5085 )
5086 .unwrap();
5087 let mut partial = f.run("p004", false).await;
5088 let reviewer = partial.worktree_root().join("review-1");
5089 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
5090 .await
5091 .unwrap();
5092 partial.retired_to = Some(authority.id.clone());
5094 partial.save_under(&f.home).unwrap();
5095 assert!(
5096 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
5097 .await
5098 .unwrap()
5099 );
5100 let git_checkpoint = f.run("p005", false).await;
5101 let location = git::git(
5102 &git_checkpoint.candidates[0].worktree,
5103 &[
5104 "rev-parse",
5105 "--path-format=absolute",
5106 "--git-path",
5107 "sequencer",
5108 ],
5109 )
5110 .await
5111 .unwrap();
5112 std::fs::create_dir(location.trim()).unwrap();
5113 let mut preservation = f.run("p006", false).await;
5114 preservation
5115 .candidate_preservation
5116 .insert('A', "pending preservation".into());
5117 preservation.save_under(&f.home).unwrap();
5118 let task = f.task(&[
5119 preservation.clone(),
5120 git_checkpoint.clone(),
5121 authority.clone(),
5122 question_run.clone(),
5123 checkpoint.clone(),
5124 retained.clone(),
5125 partial.clone(),
5126 ]);
5127 let questions = crate::ask::Questions::at(f.home.join("questions"));
5128 let mut question = crate::ask::Question::new(
5129 question_run.id.clone(),
5130 "implement".into(),
5131 "fixture".into(),
5132 "owner decision".into(),
5133 String::new(),
5134 vec!["keep".into()],
5135 );
5136 questions.put(&mut question).unwrap();
5137 let mut report = Report::default();
5138 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5139 assert_eq!(report.retired, 1, "{report:?}");
5140 assert!(!reviewer.exists());
5141 for state in [
5142 &question_run,
5143 &checkpoint,
5144 &retained,
5145 &git_checkpoint,
5146 &preservation,
5147 ] {
5148 assert!(state.candidates[0].worktree.exists());
5149 assert!(
5150 RunState::load_under(&state.id, &f.home)
5151 .unwrap()
5152 .retired_to
5153 .is_none()
5154 );
5155 }
5156 assert!(questions.get(&question.id).unwrap().status.open());
5157 assert_eq!(
5158 crate::queue::Queue::at(f.home.join("queue"))
5159 .get(&task.id)
5160 .unwrap()
5161 .status,
5162 task.status
5163 );
5164 }
5165
5166 #[tokio::test]
5167 async fn preservation_schema_migration_remains_distinct_from_retirement() {
5168 let f = Fx::new().await;
5169 let mut state = f.run("p007", false).await;
5170 state.schema = 15;
5171 state.save_under(&f.home).unwrap();
5172 let native = RunState::load_under(&state.id, &f.home).unwrap();
5173 assert_eq!(native.schema, crate::run::SCHEMA);
5174 assert!(native.candidate_preservation.is_empty());
5175 assert!(native.retired_to.is_none());
5176 state.schema = 14;
5177 state.save_under(&f.home).unwrap();
5178 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
5179 assert!(legacy.candidate_preservation.contains_key(&'A'));
5180 legacy.save_under(&f.home).unwrap();
5181 let restored = RunState::load_under(&state.id, &f.home).unwrap();
5182 assert_eq!(
5183 restored.candidate_preservation,
5184 legacy.candidate_preservation
5185 );
5186 }
5187
5188 #[tokio::test]
5189 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
5190 let f = Fx::new().await;
5191 let cache = f.repo.join("cache");
5192 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
5193 std::fs::create_dir_all(&cache).unwrap();
5194 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
5195 assert!(
5196 crate::cache::maintenance_prune(&f.home, &cache, 1)
5197 .unwrap()
5198 .is_none()
5199 );
5200 drop(guard);
5201 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5202 }
5203
5204 #[tokio::test]
5205 async fn launcher_death_does_not_release_live_tool_output_reservations() {
5206 let f = Fx::new().await;
5207 let run = f.run("abcd", false).await;
5208 let wt = &run.candidates[0].worktree;
5209 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
5210 helper
5211 .quiet()
5212 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
5213 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
5214 .env("MAGI_TEST_WORKTREE", wt)
5215 .stdin(std::process::Stdio::null())
5216 .stdout(std::process::Stdio::null())
5217 .stderr(std::process::Stdio::null())
5218 .kill_on_drop(true);
5219 assert!(
5220 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
5221 .await
5222 .unwrap()
5223 .unwrap()
5224 .success()
5225 );
5226 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
5227 .unwrap()
5228 .trim()
5229 .parse()
5230 .unwrap();
5231 struct KillTool(i32);
5232 impl Drop for KillTool {
5233 fn drop(&mut self) {
5234 unsafe { libc::kill(self.0, libc::SIGKILL) };
5236 }
5237 }
5238 let tool = KillTool(pid);
5239 assert!(OutputLease::take(wt, true).is_err());
5240 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
5241 drop(tool);
5242 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5243 loop {
5244 if OutputLease::take(wt, true).is_ok()
5245 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
5246 {
5247 break;
5248 }
5249 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5250 }
5251 })
5252 .await
5253 .unwrap();
5254 }
5255
5256 #[tokio::test]
5257 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
5258 let f = Fx::new().await;
5259 let run = f.run("abcd", false).await;
5260 let wt = &run.candidates[0].worktree;
5261 let cache = f.repo.join("cache");
5262 std::fs::create_dir(&cache).unwrap();
5263 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
5264 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
5265 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
5266 cmd.quiet()
5267 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
5268 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
5269 .stdin(std::process::Stdio::null())
5270 .stdout(std::process::Stdio::null())
5271 .stderr(std::process::Stdio::null())
5272 .kill_on_drop(true);
5273 if clear_owner {
5274 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
5275 } else {
5276 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
5277 }
5278 let gib = 1024 * 1024 * 1024;
5279 let mut leases = vec![
5280 OutputLease::borrow(wt).unwrap().unwrap(),
5281 OutputLease::cache(&f.home, &cache, false).unwrap(),
5282 ];
5283 leases.extend(
5284 OutputLease::admit_with(
5285 &f.home,
5286 std::slice::from_ref(wt),
5287 8 * gib,
5288 |_| Ok(9 * gib),
5289 )
5290 .unwrap(),
5291 );
5292 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
5293 assert!(
5294 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
5295 .await
5296 .unwrap()
5297 .unwrap()
5298 .success()
5299 );
5300 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
5301 struct KillTool(Option<i32>);
5302 impl Drop for KillTool {
5303 fn drop(&mut self) {
5304 if let Some(pid) = self.0 {
5306 unsafe { libc::kill(pid, libc::SIGKILL) };
5307 }
5308 }
5309 }
5310 let mut tool = KillTool(Some(pid));
5311 assert!(OutputLease::take(wt, true).is_err());
5315 let finish = if cancel {
5316 drop(running);
5317 None
5318 } else {
5319 Some(tokio::spawn(async move { running.finish().await }))
5320 };
5321 if clear_owner {
5322 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
5323 assert!(crate::proc::pid_alive(pid as u32));
5324 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
5325 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
5326 assert!(
5327 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
5328 9 * gib
5329 ))
5330 .is_err(),
5331 "orphan still owns its disk reservation"
5332 );
5333 assert!(
5334 crate::cache::maintenance_prune(&f.home, &cache, 0)
5335 .unwrap()
5336 .is_none()
5337 );
5338 if let Some(finish) = &finish {
5339 assert!(!finish.is_finished());
5340 }
5341 unsafe { libc::kill(pid, libc::SIGKILL) };
5343 }
5344 if let Some(finish) = finish {
5345 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
5346 .await
5347 .unwrap()
5348 .unwrap();
5349 }
5350 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5351 loop {
5352 if OutputLease::take(wt, true).is_ok()
5353 && OutputLease::cache(&f.home, &cache, true).is_ok()
5354 && OutputLease::admit_with(
5355 &f.home,
5356 std::slice::from_ref(wt),
5357 8 * gib,
5358 |_| Ok(9 * gib),
5359 )
5360 .is_ok()
5361 {
5362 if !clear_owner {
5364 assert!(!crate::proc::pid_alive(pid as u32));
5365 }
5366 break;
5367 }
5368 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5369 }
5370 })
5371 .await
5372 .unwrap();
5373 tool.0 = None;
5374 }
5375 }
5376
5377 #[tokio::test]
5378 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
5379 let f = Fx::new().await;
5380 let run = f.run("abcd", false).await;
5381 let wt = &run.candidates[0].worktree;
5382 let cache = f.repo.join("cache");
5383 std::fs::create_dir(&cache).unwrap();
5384 let pid_file = f.repo.join("child-pid");
5385 let mut cmd = tokio::process::Command::new("sh");
5386 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
5387 .quiet()
5388 .env("CHILD_PID_FILE", &pid_file)
5389 .process_group(0)
5390 .kill_on_drop(true);
5391 let running = crate::seat_process::Running::spawn(
5392 cmd,
5393 None,
5394 vec![
5395 OutputLease::borrow(wt).unwrap().unwrap(),
5396 OutputLease::cache(&f.home, &cache, false).unwrap(),
5397 ],
5398 )
5399 .unwrap();
5400 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5401 while !pid_file.exists() {
5402 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5403 }
5404 })
5405 .await
5406 .unwrap();
5407 assert!(OutputLease::take(wt, true).is_err());
5408 assert!(
5409 crate::cache::maintenance_prune(&f.home, &cache, 0)
5410 .unwrap()
5411 .is_none()
5412 );
5413 let pid: u32 = std::fs::read_to_string(pid_file)
5414 .unwrap()
5415 .trim()
5416 .parse()
5417 .unwrap();
5418 drop(running);
5419 tokio::time::timeout(std::time::Duration::from_secs(10), async {
5420 loop {
5421 if let Ok(guard) = OutputLease::take(wt, true) {
5422 assert!(
5423 !crate::proc::pid_alive(pid),
5424 "output lock released before tool death"
5425 );
5426 drop(guard);
5427 break;
5428 }
5429 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5430 }
5431 })
5432 .await
5433 .unwrap();
5434 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5435 }
5436
5437 #[tokio::test]
5438 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
5439 let f = Fx::new().await;
5440 let reason = tokio::time::timeout(
5441 std::time::Duration::from_secs(2),
5442 emergency(vec![f.repo.clone()], u64::MAX),
5443 )
5444 .await
5445 .unwrap();
5446 assert!(reason.contains("disk pressure"));
5447 assert!(
5448 tokio::time::timeout(
5449 std::time::Duration::from_millis(10),
5450 emergency(vec![f.repo.clone()], 0)
5451 )
5452 .await
5453 .is_err()
5454 );
5455 assert_eq!(
5456 std::fs::read_to_string(f.repo.join("source")).unwrap(),
5457 "base\n"
5458 );
5459 }
5460
5461 #[tokio::test]
5462 async fn automatic_requeues_cannot_refund_failed_input_generation() {
5463 let f = Fx::new().await;
5464 let mut run = f.run("1234", false).await;
5465 run.prep_failure = Some("cherry-pick conflict".into());
5466 let mut task = f.task(std::slice::from_ref(&run));
5467 let inputs = crate::queue::recovery_inputs(&run.sources);
5468 for _ in 0..108 {
5469 task.recovery_failed(&run);
5470 task.requeue();
5471 assert_eq!(task.attempts, 0);
5472 }
5473 let bytes = serde_json::to_vec(&task).unwrap();
5474 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
5475 assert!(
5476 restored
5477 .recovery_exhausted(&run.base_commit, &inputs, 2)
5478 .is_some()
5479 );
5480 assert!(
5481 restored
5482 .recovery_exhausted("different-base", &inputs, 2)
5483 .is_none()
5484 );
5485 let original_base = run.base_commit.clone();
5486 run.base_commit = "different-base".into();
5487 restored.recovery_failed(&run);
5488 assert!(
5489 restored
5490 .recovery_exhausted(&original_base, &inputs, 2)
5491 .is_some(),
5492 "returning to old inputs does not replenish their budget"
5493 );
5494 assert!(
5495 restored
5496 .recovery_exhausted(&run.base_commit, &inputs, 2)
5497 .is_none()
5498 );
5499 restored.release_operator("test-owner", None);
5500 assert!(
5501 restored
5502 .recovery_exhausted(&run.base_commit, &inputs, 2)
5503 .is_none()
5504 );
5505 }
5506 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5507 let mut state = f.run(tag, useful).await;
5508 state.status = RunStatus::Superseded;
5509 state.save_under(&f.home).unwrap();
5510 state
5511 }
5512
5513 fn historical_review(state: &mut RunState, head: &str) {
5514 state.reviews.push(
5515 serde_json::from_value(serde_json::json!({
5516 "round": state.reviews.len() + 1, "head": head, "reviews": []
5517 }))
5518 .unwrap(),
5519 );
5520 }
5521
5522 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5523 let path = state.worktree_root().join("review-1");
5524 git::worktree_add_detached(&f.repo, &path, head)
5525 .await
5526 .unwrap();
5527 path
5528 }
5529
5530 #[tokio::test]
5531 async fn source_eviction_separates_run_authority_from_read_seats() {
5532 for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5533 let f = Fx::new().await;
5534 let mut state = inactive_source(&f, "ef01", true).await;
5535 state.status = status;
5536 state.pr = Some(
5537 serde_json::from_value(serde_json::json!({
5538 "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5539 "checks":"green", "round":1, "rounds":1
5540 }))
5541 .unwrap(),
5542 );
5543 state.pr_head = Some(state.candidates[0].branch.clone());
5544 state.pr_merged_head = Some(f.base.clone());
5545 state.recovery = Some(
5546 serde_json::from_value(serde_json::json!({
5547 "from_run": "20260101-000000-abcd", "generation": 1,
5548 "mode": "full-review"
5549 }))
5550 .unwrap(),
5551 );
5552 state.pr_recovery = Some(
5553 serde_json::from_value(serde_json::json!({
5554 "url": "https://example.invalid/pull/1", "branch": "public",
5555 "head": f.base, "source_run": state.id,
5556 "instruction": "keep implementation", "approved_at": Fx::now()
5557 }))
5558 .unwrap(),
5559 );
5560 state.save_under(&f.home).unwrap();
5561 let review = reviewer(&f, &state, &f.base).await;
5562 let mut other = inactive_source(&f, "ef02", true).await;
5563 other.pr_recovery = state.pr_recovery.clone();
5564 other.save_under(&f.home).unwrap();
5565 let mut task = f.task(&[state.clone(), other]);
5566 task.pr_recovery = state.pr_recovery.clone();
5567 crate::queue::Queue::at(f.home.join("queue"))
5568 .put(&mut task)
5569 .unwrap();
5570 let run_path = f.home.join("runs").join(&state.id).join("run.json");
5571 let before = std::fs::read(&run_path).unwrap();
5572 let task_before = serde_json::to_value(&task).unwrap();
5573 let mut report = Report::default();
5574 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5575 assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5576 assert!(!review.exists());
5577 assert!(state.candidates[0].worktree.exists());
5578 assert!(
5579 git::branch_exists(&f.repo, &state.candidates[0].branch)
5580 .await
5581 .unwrap()
5582 );
5583 assert_eq!(std::fs::read(&run_path).unwrap(), before);
5584 assert_eq!(
5585 serde_json::to_value(
5586 crate::queue::Queue::at(f.home.join("queue"))
5587 .get(&task.id)
5588 .unwrap()
5589 )
5590 .unwrap(),
5591 task_before
5592 );
5593 }
5594 }
5595
5596 #[tokio::test]
5597 async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5598 for kind in ["own-path", "active-seat", "paused-session"] {
5599 let f = Fx::new().await;
5600 let mut state = inactive_source(&f, "ef03", true).await;
5601 let review = reviewer(&f, &state, &f.base).await;
5602 let expected = match kind {
5603 "own-path" => {
5604 state.attachments.push(review.join("source"));
5605 "run.attachments[0]"
5606 }
5607 "active-seat" => {
5608 state.active.insert(
5609 "review-1".into(),
5610 serde_json::from_value(serde_json::json!({
5611 "node":"review", "started_at":Fx::now(), "timeout_secs":60
5612 }))
5613 .unwrap(),
5614 );
5615 "continuation lease"
5616 }
5617 _ => {
5618 state.status = RunStatus::Blocked;
5619 state.parked = true;
5620 let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5621 seat.turns = 2;
5622 seat.captured_session = Some("saved-session".into());
5623 state.seats.insert("review-1".into(), seat);
5624 "resume contract"
5625 }
5626 };
5627 state.save_under(&f.home).unwrap();
5628 let mut report = Report::default();
5629 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5630 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5631 assert!(review.exists());
5632 assert!(
5633 report.retained.iter().any(|r| r.contains(expected)),
5634 "{kind}: {report:?}"
5635 );
5636 restore_execution_seats(&f.home, &state).await.unwrap();
5639 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5640 }
5641 }
5642
5643 #[tokio::test]
5644 async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5645 for kind in ["waiting", "lease", "historical", "unknown"] {
5646 let f = Fx::new().await;
5647 let state = inactive_source(&f, "ef06", true).await;
5648 let review = reviewer(&f, &state, &f.base).await;
5649 let store = crate::ask::Questions::at(f.home.join("questions"));
5650 let mut question = crate::ask::Question::new(
5651 "20260101-000000-ffff".into(),
5652 "review".into(),
5653 "review-1".into(),
5654 "fixture".into(),
5655 "fixture".into(),
5656 Vec::new(),
5657 );
5658 question.cwd = Some(review.to_string_lossy().into_owned());
5659 if kind != "waiting" {
5660 question.status = crate::ask::QuestionStatus::Abandoned;
5661 }
5662 store.put(&mut question).unwrap();
5663 if kind == "lease" {
5664 store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5665 }
5666 if kind == "unknown" {
5667 let path = store.path_of(&question.id);
5668 let mut raw: serde_json::Value =
5669 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5670 raw["future_cwd_authority"] = serde_json::Value::Null;
5671 std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5672 }
5673 let mut report = Report::default();
5674 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5675 assert_eq!(
5676 report.evicted_copies,
5677 usize::from(kind == "historical"),
5678 "{kind}: {report:?}"
5679 );
5680 assert_eq!(review.exists(), kind != "historical");
5681 if kind == "waiting" || kind == "lease" {
5682 assert!(
5683 report
5684 .retained
5685 .iter()
5686 .any(|r| r.contains("question") && r.contains("cwd")),
5687 "{report:?}"
5688 );
5689 }
5690 }
5691 }
5692
5693 #[tokio::test]
5694 async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5695 let f = Fx::new().await;
5696 let mut state = inactive_source(&f, "ef07", true).await;
5697 let review = reviewer(&f, &state, &f.base).await;
5698 let mut report = Report::default();
5699 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5700 assert_eq!(report.evicted_copies, 1, "{report:?}");
5701 state.config.worktree.setup = vec![crate::config::SetupStep {
5702 run: Some("printf preserved > source; exit 1".into()),
5703 ..Default::default()
5704 }];
5705 state.save_under(&f.home).unwrap();
5706 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5707 let marker = f
5708 .home
5709 .join("runs")
5710 .join(&state.id)
5711 .join("artifacts/restore-review-1.pending");
5712 assert!(marker.exists());
5713 assert_eq!(
5714 std::fs::read_to_string(review.join("source")).unwrap(),
5715 "preserved"
5716 );
5717 assert!(
5718 restore_execution_seats(&f.home, &state)
5719 .await
5720 .unwrap_err()
5721 .to_string()
5722 .contains("interrupted execution restoration")
5723 );
5724 let mut retry = Report::default();
5725 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5726 assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5727 assert!(
5728 retry
5729 .retained
5730 .iter()
5731 .any(|r| r.contains("restoration checkpoint")),
5732 "{retry:?}"
5733 );
5734 assert!(marker.exists());
5735 assert_eq!(
5736 std::fs::read_to_string(review.join("source")).unwrap(),
5737 "preserved"
5738 );
5739 }
5740
5741 #[tokio::test]
5742 async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5743 for kind in [
5744 "alternates",
5745 "promisor-config",
5746 "promisor-pack",
5747 "objects-symlink",
5748 ] {
5749 let f = Fx::new().await;
5750 let state = inactive_source(&f, "ef11", true).await;
5751 let review = reviewer(&f, &state, &f.base).await;
5752 let common = crate::repo_lock::common_dir(&f.repo)
5753 .await
5754 .unwrap()
5755 .unwrap();
5756 match kind {
5757 "alternates" => {
5758 let borrowed = f._temp.path().join("borrowed-objects");
5759 std::fs::create_dir(&borrowed).unwrap();
5760 std::fs::write(
5761 common.join("objects/info/alternates"),
5762 format!("{}\n", borrowed.display()),
5763 )
5764 .unwrap();
5765 }
5766 "promisor-config" => {
5767 git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5768 .await
5769 .unwrap();
5770 }
5771 "promisor-pack" => {
5772 std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5773 }
5774 _ => {
5775 let borrowed = f._temp.path().join("borrowed-objects");
5776 std::fs::rename(common.join("objects"), &borrowed).unwrap();
5777 std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5778 }
5779 }
5780 let mut report = Report::default();
5781 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5782 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5783 assert!(review.exists());
5784 assert!(
5785 report.retained.iter().any(|r| r.contains("source anchor")),
5786 "{kind}: {report:?}"
5787 );
5788 }
5789 }
5790
5791 #[tokio::test]
5792 async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5793 let mut f = Fx::new().await;
5794 crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5795 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5796 assert_eq!(
5797 git::git(&f.repo, &["rev-list", "--count", &f.base])
5798 .await
5799 .unwrap(),
5800 "5001"
5801 );
5802 let state = inactive_source(&f, "ef10", true).await;
5803 let review = reviewer(&f, &state, &f.base).await;
5804 std::fs::write(review.join("source"), "bounded review delta").unwrap();
5805 git::git(&review, &["add", "."]).await.unwrap();
5806 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5807 let head = git::git(
5808 &review,
5809 &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5810 )
5811 .await
5812 .unwrap();
5813 git::git(&review, &["checkout", "--detach", &head])
5814 .await
5815 .unwrap();
5816 let run_file = f.home.join("runs").join(&state.id).join("run.json");
5817 let run_before = std::fs::read(&run_file).unwrap();
5818 let mut report = Report::default();
5819 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5820 assert_eq!(report.evicted_copies, 1, "{report:?}");
5821 assert!(!review.exists());
5822 assert!(state.candidates[0].worktree.exists());
5823 assert!(
5824 git::branch_exists(&f.repo, &state.candidates[0].branch)
5825 .await
5826 .unwrap()
5827 );
5828 assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5829 let archive = f
5830 .home
5831 .join("runs")
5832 .join(&state.id)
5833 .join("artifacts/source-evidence")
5834 .join(format!("{head}-{}", f.base));
5835 let manifest: serde_json::Value =
5836 serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5837 assert_eq!(manifest["anchor"], f.base);
5838 assert!(
5839 std::fs::metadata(archive.join("objects.pack"))
5840 .unwrap()
5841 .len()
5842 < 16 * 1024
5843 );
5844 let pin = format!("refs/magi/source-anchors/{}", f.base);
5845 assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5846 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5847 .await
5848 .unwrap();
5849 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5850 assert!(
5851 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5852 .await
5853 .unwrap()
5854 .ok()
5855 );
5856 let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5857 .await
5858 .unwrap();
5859 for kind in ["missing", "moved", "symbolic", "symlink"] {
5861 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5862 .await
5863 .unwrap();
5864 match kind {
5865 "moved" => {
5866 git::git(&f.repo, &["update-ref", &pin, &other])
5867 .await
5868 .unwrap();
5869 }
5870 "symbolic" => {
5871 git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5872 .await
5873 .unwrap();
5874 }
5875 "symlink" => {
5876 let common = crate::repo_lock::common_dir(&f.repo)
5877 .await
5878 .unwrap()
5879 .unwrap();
5880 let target = f._temp.path().join("operator-anchor");
5881 std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5882 let file = common.join(&pin);
5883 std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5884 std::os::unix::fs::symlink(target, file).unwrap();
5885 }
5886 _ => {}
5887 }
5888 assert!(
5889 restore_execution_seats(&f.home, &state).await.is_err(),
5890 "{kind}"
5891 );
5892 assert!(!review.exists());
5893 assert!(
5894 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5895 .await
5896 .unwrap()
5897 .ok()
5898 );
5899 if kind == "symlink" {
5900 let common = crate::repo_lock::common_dir(&f.repo)
5901 .await
5902 .unwrap()
5903 .unwrap();
5904 std::fs::remove_file(common.join(&pin)).unwrap();
5905 } else {
5906 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5907 .await
5908 .unwrap();
5909 }
5910 git::git(&f.repo, &["update-ref", &pin, &f.base])
5911 .await
5912 .unwrap();
5913 }
5914 assert!(
5917 git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5918 .await
5919 .unwrap()
5920 );
5921 git::git(
5922 &f.repo,
5923 &[
5924 "update-ref",
5925 "-d",
5926 &format!("refs/heads/{}", state.candidates[0].branch),
5927 ],
5928 )
5929 .await
5930 .unwrap();
5931 git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5932 .await
5933 .unwrap();
5934 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5935 .await
5936 .unwrap();
5937 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5938 assert_eq!(
5939 git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5940 .await
5941 .unwrap(),
5942 pin
5943 );
5944 assert_eq!(
5945 git::git(&f.repo, &["rev-list", "--count", &f.base])
5946 .await
5947 .unwrap(),
5948 "5001"
5949 );
5950 restore_execution_seats(&f.home, &state).await.unwrap();
5951 assert_eq!(
5952 git::git(&review, &["rev-list", "--count", "HEAD"])
5953 .await
5954 .unwrap(),
5955 "5002"
5956 );
5957 assert_eq!(
5958 git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5959 .await
5960 .unwrap(),
5961 "false"
5962 );
5963 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5964 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5965 let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5966 assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5967 std::fs::remove_dir_all(&f.repo).unwrap();
5968 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5969 let inspection = f._temp.path().join("shared-history-inspection");
5970 restore_seat(&f.home, &state.id, &head, &inspection)
5971 .await
5972 .unwrap();
5973 assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5974 assert_eq!(
5975 git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5976 .await
5977 .unwrap(),
5978 "true"
5979 );
5980 assert_eq!(
5981 git::git(&inspection, &["log", "--format=%H"])
5982 .await
5983 .unwrap(),
5984 format!("{head}\n{}", f.base)
5985 );
5986 assert!(!inspection.join(".git/objects/info/alternates").exists());
5987 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5988 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5989 }
5990
5991 #[tokio::test]
5992 async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5993 let f = Fx::new().await;
5994 let state = inactive_source(&f, "ef08", true).await;
5995 let review = reviewer(&f, &state, &f.base).await;
5996 let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5997 let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5998 .await
5999 .unwrap();
6000 let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
6001 .await
6002 .unwrap();
6003 git::git(&review, &["checkout", "--detach", &head])
6004 .await
6005 .unwrap();
6006 let archive =
6007 crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
6008 let mut report = Report::default();
6009 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6010 assert_eq!(report.evicted_copies, 0, "{report:?}");
6011 assert!(review.exists());
6012 assert!(
6013 report
6014 .retained
6015 .iter()
6016 .any(|r| r.contains("incomplete archived commit ancestry")),
6017 "{report:?}"
6018 );
6019 assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
6021 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6022 .await
6023 .unwrap();
6024 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6025 for oid in [&parent, &head] {
6026 assert!(
6027 !git::git_raw(&f.repo, &["cat-file", "-e", oid])
6028 .await
6029 .unwrap()
6030 .ok()
6031 );
6032 }
6033 let patch = source_diff(&archive).await.unwrap();
6034 assert!(
6035 crate::fold_evidence::import_snapshot(&archive, &f.repo)
6036 .await
6037 .is_err()
6038 );
6039 assert!(
6040 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6041 .await
6042 .unwrap()
6043 .ok()
6044 );
6045 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
6046 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6047 let inspection = f._temp.path().join("legacy-tree-inspection");
6048 restore_source(&archive, &inspection).await.unwrap();
6049 assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
6050 assert!(
6051 !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
6052 .await
6053 .unwrap()
6054 .ok()
6055 );
6056 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
6057 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
6058 assert_eq!(source_diff(&archive).await.unwrap(), patch);
6059 assert!(state.candidates[0].worktree.exists());
6060 }
6061
6062 #[tokio::test]
6063 async fn source_eviction_retains_ancestry_above_expanded_budget() {
6064 let f = Fx::new().await;
6065 let state = inactive_source(&f, "ef09", true).await;
6066 let review = reviewer(&f, &state, &f.base).await;
6067 std::fs::File::create(review.join("ancestor-only"))
6068 .unwrap()
6069 .set_len(64 * 1024 * 1024 + 1)
6070 .unwrap();
6071 git::git(&review, &["add", "."]).await.unwrap();
6072 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
6073 let parent = git::git(
6074 &review,
6075 &["commit-tree", &parent_tree, "-m", "large ancestor"],
6076 )
6077 .await
6078 .unwrap();
6079 std::fs::remove_file(review.join("ancestor-only")).unwrap();
6080 git::git(&review, &["add", "."]).await.unwrap();
6081 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6082 let head = git::git(
6083 &review,
6084 &[
6085 "commit-tree",
6086 &tree,
6087 "-p",
6088 &parent,
6089 "-m",
6090 "small final tree",
6091 ],
6092 )
6093 .await
6094 .unwrap();
6095 git::git(&review, &["checkout", "--detach", &head])
6096 .await
6097 .unwrap();
6098 let mut report = Report::default();
6099 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6100 assert_eq!(report.evicted_copies, 0, "{report:?}");
6101 assert!(
6102 report
6103 .retained
6104 .iter()
6105 .any(|r| r.contains("expanded object limit exceeded")),
6106 "{report:?}"
6107 );
6108 assert!(review.exists());
6109 assert!(state.candidates[0].worktree.exists());
6110 }
6111
6112 #[tokio::test]
6113 async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
6114 let f = Fx::new().await;
6115 let state = inactive_source(&f, "ef04", true).await;
6116 let review = reviewer(&f, &state, &f.base).await;
6117 std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
6118 git::git(&review, &["add", "."]).await.unwrap();
6119 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
6120 let parent = git::git(
6121 &review,
6122 &["commit-tree", &parent_tree, "-m", "unrelated parent"],
6123 )
6124 .await
6125 .unwrap();
6126 std::fs::remove_file(review.join("ancestor-only")).unwrap();
6127 std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
6128 git::git(&review, &["add", "."]).await.unwrap();
6129 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6130 let head = git::git(
6131 &review,
6132 &[
6133 "commit-tree",
6134 &tree,
6135 "-p",
6136 &parent,
6137 "-m",
6138 "unrelated review checkpoint",
6139 ],
6140 )
6141 .await
6142 .unwrap();
6143 git::git(&review, &["checkout", "--detach", &head])
6144 .await
6145 .unwrap();
6146 let mut report = Report::default();
6147 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6148 assert_eq!(report.evicted_copies, 1, "{report:?}");
6149 let ledger = f
6150 .home
6151 .join("runs")
6152 .join(&state.id)
6153 .join("artifacts/eviction-review-1.json");
6154 let original = std::fs::read(&ledger).unwrap();
6155 let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
6156 assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
6157 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6158 .await
6159 .unwrap();
6160 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6161 assert!(
6162 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6163 .await
6164 .unwrap()
6165 .ok()
6166 );
6167 assert!(
6168 !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
6169 .await
6170 .unwrap()
6171 .ok()
6172 );
6173 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
6174 assert!(
6175 seat_heads(&f.home, &state.id)
6176 .await
6177 .unwrap()
6178 .contains_key(&head)
6179 );
6180 restore_execution_seats(&f.home, &state).await.unwrap();
6181 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
6182 assert_eq!(
6183 git::git(&review, &["log", "--format=%H"]).await.unwrap(),
6184 format!("{head}\n{parent}")
6185 );
6186 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
6187 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6188 let inspection = f._temp.path().join("complete-history-inspection");
6189 restore_seat(&f.home, &state.id, &head, &inspection)
6190 .await
6191 .unwrap();
6192 assert_eq!(
6193 git::git(&inspection, &["log", "--format=%H"])
6194 .await
6195 .unwrap(),
6196 format!("{head}\n{parent}")
6197 );
6198 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
6199 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
6200
6201 assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
6202 assert!(
6203 !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
6204 .await
6205 .unwrap()
6206 .ok()
6207 );
6208 assert_eq!(
6209 std::fs::read_to_string(review.join("source")).unwrap(),
6210 "unrelated review checkpoint"
6211 );
6212 std::fs::write(review.join("operator-notes"), "keep").unwrap();
6213 assert!(restore_execution_seats(&f.home, &state).await.is_err());
6214 assert_eq!(
6215 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
6216 "keep"
6217 );
6218 std::fs::remove_file(review.join("operator-notes")).unwrap();
6219 restore_execution_seats(&f.home, &state).await.unwrap();
6220 let mut second = Report::default();
6221 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
6222 assert_eq!(second.evicted_copies, 1, "{second:?}");
6223 assert_eq!(std::fs::read(&ledger).unwrap(), original);
6224 assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
6225 let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
6228 legacy["version"] = 1.into();
6229 legacy.as_object_mut().unwrap().remove("base");
6230 std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
6231 let mut advanced = state.clone();
6232 advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6233 .await
6234 .unwrap();
6235 advanced.save_under(&f.home).unwrap();
6236 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
6237 let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
6239 let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
6240 let patch = std::fs::read(archive.join("change.patch")).unwrap();
6241 std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
6242 assert!(restore_execution_seats(&f.home, &state).await.is_err());
6243 assert!(!review.exists());
6244 std::fs::write(archive.join("change.patch"), patch).unwrap();
6245 std::fs::create_dir(&review).unwrap();
6246 std::fs::write(review.join("operator-notes"), "keep").unwrap();
6247 assert!(restore_execution_seats(&f.home, &state).await.is_err());
6248 assert_eq!(
6249 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
6250 "keep"
6251 );
6252 }
6253
6254 #[tokio::test]
6255 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
6256 use std::os::unix::fs::PermissionsExt;
6257 let mut f = Fx::new().await;
6258 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
6259 git::git(&f.repo, &["add", "."]).await.unwrap();
6260 git::git(&f.repo, &["commit", "-m", "binary base"])
6261 .await
6262 .unwrap();
6263 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
6264 let mut state = inactive_source(&f, "ec01", true).await;
6265 let candidate = state.candidates[0].worktree.clone();
6266 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
6267 historical_review(&mut state, &earlier);
6268 let review = reviewer(&f, &state, &earlier).await;
6269 std::fs::remove_file(candidate.join("source")).unwrap();
6270 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
6271 std::fs::write(candidate.join("script"), "run\n").unwrap();
6272 std::fs::set_permissions(
6273 candidate.join("script"),
6274 std::fs::Permissions::from_mode(0o755),
6275 )
6276 .unwrap();
6277 git::git(&candidate, &["add", "."]).await.unwrap();
6278 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
6279 .await
6280 .unwrap();
6281 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
6282 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
6283 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
6284 .await
6285 .unwrap();
6286 historical_review(&mut state, &head);
6287 historical_review(&mut state, &head);
6288 state.status = RunStatus::Superseded;
6289 state.save_under(&f.home).unwrap();
6290 let task = f.task(std::slice::from_ref(&state));
6291 let before = serde_json::to_value(&state).unwrap();
6292 git::git(&review, &["reset", "--hard", &head])
6293 .await
6294 .unwrap();
6295 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6296 assert_eq!(report.evicted_copies, 1, "{report:?}");
6297 assert!(!review.exists());
6298 assert!(candidate.exists(), "selected implementation survives");
6299 assert!(report.source_unlinked_logical_bytes > 0);
6300 assert!(report.source_unlinked_allocated_bytes.is_some());
6301 assert!(report.source_evidence_added_allocated_bytes.is_some());
6302 assert_eq!(
6303 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
6304 before
6305 );
6306 assert_eq!(
6307 crate::queue::Queue::at(f.home.join("queue"))
6308 .get(&task.id)
6309 .unwrap()
6310 .runs,
6311 task.runs
6312 );
6313 let archive = f
6314 .home
6315 .join("runs")
6316 .join(&state.id)
6317 .join("artifacts/source-evidence")
6318 .join(format!("{head}-{}", f.base));
6319 let patch = source_diff(&archive).await.unwrap();
6320 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
6321 assert!(String::from_utf8_lossy(&patch).contains("100755"));
6322 assert!(
6323 git::worktree_remove_clean(&f.repo, &candidate)
6324 .await
6325 .unwrap()
6326 );
6327 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
6328 .await
6329 .unwrap();
6330 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6331 .await
6332 .unwrap();
6333 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6334 assert!(
6335 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6336 .await
6337 .unwrap()
6338 .ok()
6339 );
6340 std::fs::remove_dir_all(&f.repo).unwrap();
6342 assert_eq!(source_diff(&archive).await.unwrap(), patch);
6343 let restored = f._temp.path().join("restored");
6344 restore_source(&archive, &restored).await.unwrap();
6345 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
6346 assert_eq!(
6347 std::fs::read(restored.join("binary")).unwrap(),
6348 b"new\0binary\xfe"
6349 );
6350 assert!(!restored.join("source").exists());
6351 assert_ne!(
6352 std::fs::metadata(restored.join("script"))
6353 .unwrap()
6354 .permissions()
6355 .mode()
6356 & 0o111,
6357 0
6358 );
6359 assert!(restore_source(&archive, &restored).await.is_err());
6360 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
6363 .await
6364 .unwrap();
6365 let patch_file = f._temp.path().join("saved.patch");
6366 std::fs::write(&patch_file, &patch).unwrap();
6367 git::git(
6368 &restored,
6369 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
6370 )
6371 .await
6372 .unwrap();
6373 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
6374 assert_eq!(
6375 std::fs::read(restored.join("binary")).unwrap(),
6376 b"new\0binary\xfe"
6377 );
6378 }
6379
6380 #[tokio::test]
6381 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
6382 let f = Fx::new().await;
6383 let state = inactive_source(&f, "ec10", true).await;
6384 let review = reviewer(&f, &state, &f.base).await;
6385 let mut successor = inactive_source(&f, "ec11", true).await;
6386 successor.retired_to = Some(state.id.clone());
6387 successor.save_under(&f.home).unwrap();
6388 f.task(&[state.clone(), successor]);
6389
6390 let mut tips = Vec::new();
6391 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
6392 std::fs::write(review.join("source"), contents).unwrap();
6393 git::git(&review, &["add", "."]).await.unwrap();
6394 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6395 let tip = git::git(
6396 &review,
6397 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
6398 )
6399 .await
6400 .unwrap();
6401 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
6402 tips.push(tip);
6403 git::git(&review, &["reset", "--hard", &f.base])
6404 .await
6405 .unwrap();
6406 }
6407 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
6409 .await
6410 .unwrap();
6411 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
6412 .await
6413 .unwrap();
6414 let mut report = Report::default();
6415 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6416 assert_eq!(report.evicted_copies, 1, "{report:?}");
6417 assert!(!review.exists());
6418 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6419 .await
6420 .unwrap();
6421 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6422 for tip in &tips {
6423 assert!(
6424 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
6425 .await
6426 .unwrap()
6427 .ok()
6428 );
6429 }
6430 std::fs::remove_dir_all(&f.repo).unwrap();
6432 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
6433 for (i, tip) in tips.iter().enumerate() {
6434 assert!(
6435 shown
6436 .iter()
6437 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
6438 );
6439 let destination = f._temp.path().join(format!("restore-history-{i}"));
6440 restore_seat(&f.home, &state.id, tip, &destination)
6441 .await
6442 .unwrap();
6443 assert_eq!(
6444 std::fs::read_to_string(destination.join("source")).unwrap(),
6445 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
6446 );
6447 }
6448 assert!(
6449 restore_seat(
6450 &f.home,
6451 &state.id,
6452 "unknown",
6453 &f._temp.path().join("missing")
6454 )
6455 .await
6456 .is_err()
6457 );
6458 }
6459
6460 #[tokio::test]
6461 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
6462 fn corrupt(path: &Path) -> Result<()> {
6463 let root = path.parent().unwrap();
6464 let marker = root.join("archive-to-corrupt");
6467 let archive = std::fs::read_to_string(&marker)?;
6468 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
6469 Ok(())
6470 }
6471 let f = Fx::new().await;
6472 let state = inactive_source(&f, "ec12", true).await;
6473 let review = reviewer(&f, &state, &f.base).await;
6474 let archive = f
6475 .home
6476 .join("runs")
6477 .join(&state.id)
6478 .join("artifacts/source-evidence")
6479 .join(format!("{}-{}", f.base, f.base));
6480 std::fs::write(
6481 state.worktree_root().join("archive-to-corrupt"),
6482 archive.to_str().unwrap(),
6483 )
6484 .unwrap();
6485 let mut report = Report::default();
6486 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
6487 assert_eq!(report.evicted_copies, 0, "{report:?}");
6488 assert!(review.exists());
6489 assert!(
6490 report
6491 .retained
6492 .iter()
6493 .any(|r| r.contains("corrupt snapshot")),
6494 "{report:?}"
6495 );
6496 }
6497
6498 #[tokio::test]
6499 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6500 for kind in [
6501 "dirty",
6502 "hidden",
6503 "ignored",
6504 "empty-directory",
6505 "untracked",
6506 "borrowed",
6507 "borrowed-symlink",
6508 "lease",
6509 "owner",
6510 "driver",
6511 "task-pin",
6512 "task-replacement",
6513 "malformed-orig-head",
6514 "grace",
6515 "resumable",
6516 "archive-symlink",
6517 "root-symlink",
6518 "checkpoint",
6519 "symlink",
6520 "retained",
6521 "unreadable",
6522 "unknown-field",
6523 "unknown-task-field",
6524 ] {
6525 let f = Fx::new().await;
6526 let mut state = inactive_source(&f, "ec02", true).await;
6527 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6528 .await
6529 .unwrap();
6530 historical_review(&mut state, &head);
6531 let review = reviewer(&f, &state, &head).await;
6532 let mut output = None;
6533 let mut owner = None;
6534 let mut driver = None;
6535 let mut disk = Fx::disk();
6536 match kind {
6537 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6538 "hidden" => {
6539 git::git(&review, &["update-index", "--assume-unchanged", "source"])
6540 .await
6541 .unwrap();
6542 std::fs::write(review.join("source"), "notes").unwrap();
6543 }
6544 "ignored" => {
6545 std::fs::create_dir(review.join("scratch")).unwrap();
6546 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6547 }
6548 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6549 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6550 "borrowed" => {
6551 let mut other = f.run("ec03", true).await;
6552 other.candidates[0].worktree = review.clone();
6553 other.save_under(&f.home).unwrap();
6554 }
6555 "borrowed-symlink" => {
6556 let alias = f._temp.path().join("borrowed-alias");
6557 std::os::unix::fs::symlink(&review, &alias).unwrap();
6558 let mut other = f.run("ec03", true).await;
6559 other.candidates[0].worktree = alias;
6560 other.save_under(&f.home).unwrap();
6561 }
6562 "lease" => output = OutputLease::borrow(&review).unwrap(),
6563 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6564 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6565 "task-pin" => {
6566 let mut task = f.task(std::slice::from_ref(&state));
6567 task.resume_override = Some(crate::queue::OperatorResume {
6568 question_id: "fixture-pin".into(),
6569 at: Fx::now(),
6570 conductor_rehold: None,
6571 forced: true,
6572 pinned_run: Some(state.id.clone()),
6573 });
6574 crate::queue::Queue::at(f.home.join("queue"))
6575 .put(&mut task)
6576 .unwrap();
6577 }
6578 "task-replacement" => {
6579 let mut task = f.task(std::slice::from_ref(&state));
6580 task.replacement = Some(crate::merge_decision::Replacement {
6581 question: "fixture-replacement".into(),
6582 run: state.id.clone(),
6583 pr: "https://example.invalid/pull/1".into(),
6584 agent: "worker".into(),
6585 context: String::new(),
6586 });
6587 crate::queue::Queue::at(f.home.join("queue"))
6588 .put(&mut task)
6589 .unwrap();
6590 }
6591 "malformed-orig-head" => {
6592 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6593 .await
6594 .unwrap();
6595 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6596 .unwrap();
6597 }
6598 "grace" => disk.fold_grace_secs = 3600,
6599 "resumable" => state.status = RunStatus::Blocked,
6600 "archive-symlink" => {
6601 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6602 std::fs::create_dir_all(&artifacts).unwrap();
6603 let outside = f._temp.path().join("outside");
6604 std::fs::create_dir(&outside).unwrap();
6605 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6606 .unwrap();
6607 }
6608 "root-symlink" => {
6609 let root = state.worktree_root();
6610 let outside = f._temp.path().join("outside-root");
6611 std::fs::rename(&root, &outside).unwrap();
6612 std::os::unix::fs::symlink(&outside, &root).unwrap();
6613 }
6614 "checkpoint" => {
6615 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6616 .await
6617 .unwrap();
6618 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6619 }
6620 "symlink" => {
6621 std::fs::remove_file(review.join("source")).unwrap();
6622 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6623 }
6624 "retained" => std::fs::write(
6625 f.home
6626 .join("runs")
6627 .join(&state.id)
6628 .join("retain-recovery-sources"),
6629 "owner",
6630 )
6631 .unwrap(),
6632 "unreadable" => {
6633 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6634 }
6635 "unknown-field" | "unknown-task-field" => {}
6636 _ => unreachable!(),
6637 }
6638 state.save_under(&f.home).unwrap();
6639 if kind == "unknown-field" {
6640 let path = f.home.join("runs").join(&state.id).join("run.json");
6641 let mut raw: serde_json::Value =
6642 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6643 raw["future_source_owner"] = serde_json::json!({"path": review});
6644 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6645 }
6646 if kind == "unknown-task-field" {
6647 let task = f.task(std::slice::from_ref(&state));
6648 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6649 let mut raw: serde_json::Value =
6650 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6651 raw["future_source_owner"] = serde_json::json!({"path": review});
6652 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6653 }
6654 let mut report = Report::default();
6655 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6656 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6657 assert!(review.exists(), "{kind}: {report:?}");
6658 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6659 drop(output);
6660 drop(owner);
6661 drop(driver);
6662 }
6663 }
6664
6665 #[tokio::test]
6666 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6667 let f = Fx::new().await;
6668 let state = inactive_source(&f, "ec04", true).await;
6669 let review = reviewer(&f, &state, &f.base).await;
6670 let mut report = Report::default();
6671 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6672 assert_eq!(report.evicted_copies, 1, "{report:?}");
6673 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6674 let ledger = artifacts.join("eviction-review-1.json");
6675 let mut record: serde_json::Value =
6676 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6677 record["phase"] = "pending".into();
6678 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6679 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6680 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6681 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6682 let mut blocked = Report::default();
6683 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6684 assert!(
6685 blocked
6686 .retained
6687 .iter()
6688 .any(|r| r.contains("corrupt snapshot")),
6689 "{blocked:?}"
6690 );
6691 assert_eq!(
6692 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6693 ["phase"],
6694 "pending"
6695 );
6696 assert!(
6697 restore_source(&archive, &f._temp.path().join("bad-restore"))
6698 .await
6699 .is_err()
6700 );
6701 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6702 let mut retried = Report::default();
6703 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6704 assert_eq!(
6705 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6706 ["phase"],
6707 "evicted"
6708 );
6709 assert_eq!(retried.evicted_copies, 0);
6710 assert!(!review.exists());
6711 let pending = archive.with_extension("pending");
6713 std::fs::rename(&archive, &pending).unwrap();
6714 assert_eq!(
6715 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6716 .await
6717 .unwrap(),
6718 archive
6719 );
6720 assert!(!pending.exists());
6721 }
6722
6723 #[tokio::test]
6724 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6725 let f = Fx::new().await;
6726 let mut empty = inactive_source(&f, "ec05", false).await;
6727 historical_review(&mut empty, &f.base);
6728 empty.status = RunStatus::Superseded;
6729 empty.save_under(&f.home).unwrap();
6730 let source = inactive_source(&f, "ec06", true).await;
6731 let task = f.task(&[empty.clone(), source.clone()]);
6732 let before = serde_json::to_value(&empty).unwrap();
6733 let mut report = Report::default();
6734 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6735 assert_eq!(report.evicted_copies, 1, "{report:?}");
6736 assert!(!empty.candidates[0].worktree.exists());
6737 assert!(source.candidates[0].worktree.exists());
6738 assert!(
6739 git::branch_exists(&f.repo, &empty.candidates[0].branch)
6740 .await
6741 .unwrap()
6742 );
6743 assert_eq!(
6744 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6745 before
6746 );
6747 let mut retry = Report::default();
6748 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6749 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6750 assert!(!empty.worktree_root().exists());
6751 assert_eq!(
6752 crate::queue::Queue::at(f.home.join("queue"))
6753 .get(&task.id)
6754 .unwrap()
6755 .runs,
6756 task.runs
6757 );
6758 }
6759 #[tokio::test]
6760 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6761 fn interrupt(_: &Path) -> Result<()> {
6762 anyhow::bail!("fixture interruption before unlink")
6763 }
6764 fn vanish(path: &Path) -> Result<()> {
6765 std::fs::remove_dir_all(path)?;
6766 anyhow::bail!("fixture interruption after checkout unlink")
6767 }
6768 for stage in [
6769 "intent",
6770 "missing-registration",
6771 "changed-admin",
6772 "missing-archive",
6773 "wrong-identity",
6774 ] {
6775 let f = Fx::new().await;
6776 let state = inactive_source(&f, "ec07", true).await;
6777 let review = reviewer(&f, &state, &f.base).await;
6778 let admin = PathBuf::from(
6779 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6780 .await
6781 .unwrap(),
6782 );
6783 let mut interrupted = Report::default();
6784 copies::evict_with(
6785 &f.home,
6786 &Fx::disk(),
6787 Fx::now(),
6788 &mut interrupted,
6789 if stage == "missing-registration" || stage == "changed-admin" {
6790 vanish
6791 } else {
6792 interrupt
6793 },
6794 )
6795 .await;
6796 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6797 let ledger = f
6798 .home
6799 .join("runs")
6800 .join(&state.id)
6801 .join("artifacts/eviction-review-1.json");
6802 let record: serde_json::Value =
6803 serde_json::from_slice(&std::fs::read(&ledger).unwrap_or_else(|error| {
6804 panic!("{stage}: missing pending ledger: {error}; {interrupted:?}")
6805 }))
6806 .unwrap();
6807 assert_eq!(record["phase"], "pending");
6808 if stage == "changed-admin" {
6809 std::fs::write(admin.join("owner-note"), "keep").unwrap();
6810 }
6811 if stage == "missing-archive" {
6812 std::fs::remove_file(
6813 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6814 )
6815 .unwrap();
6816 }
6817 if stage == "wrong-identity" {
6818 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6819 .await
6820 .unwrap();
6821 let other =
6822 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6823 .await
6824 .unwrap();
6825 let original = Path::new(record["evidence"].as_str().unwrap());
6826 for name in [
6827 "manifest.json",
6828 "objects.pack",
6829 "change.patch",
6830 "change.index",
6831 ] {
6832 std::fs::copy(other.join(name), original.join(name)).unwrap();
6833 }
6834 }
6835 let mut retried = Report::default();
6836 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6837 let phase =
6838 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6839 .unwrap()["phase"]
6840 .clone();
6841 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6842 assert_eq!(phase, "pending", "{stage}: {retried:?}");
6843 assert!(!retried.retained.is_empty());
6844 if stage == "changed-admin" {
6845 assert!(admin.join("owner-note").exists());
6846 } else {
6847 assert!(review.exists());
6848 }
6849 } else {
6850 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6851 assert!(!review.exists());
6852 assert!(!admin.exists());
6853 }
6854 }
6855 }
6856 fn source_census(home: &Path) -> serde_json::Value {
6857 serde_json::from_slice(&std::fs::read(home.join("source-eviction-cursor")).unwrap())
6858 .unwrap()
6859 }
6860
6861 #[tokio::test]
6862 async fn source_eviction_scan_bounds_missing_root_checkpoints_and_eventually_reclaims() {
6863 let f = Fx::new().await;
6864 let state = inactive_source(&f, "ffff", true).await;
6865 let review = reviewer(&f, &state, &f.base).await;
6866 for i in 0..130 {
6867 let mut old = state.clone();
6868 old.id = format!("20260101-000000-{i:04x}");
6869 old.candidates.clear();
6870 old.save_under(&f.home).unwrap();
6871 if i == 0 {
6873 let artifacts = f.home.join("runs").join(&old.id).join("artifacts");
6874 std::fs::create_dir_all(&artifacts).unwrap();
6875 std::fs::write(artifacts.join("eviction-review-1.json"), "{}").unwrap();
6876 }
6877 }
6878 for cycle in 1..=2 {
6883 for (pass, expected_next) in [65, 129, 131].into_iter().enumerate() {
6884 let mut report = Report::default();
6885 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6886 assert!(report.failures.is_empty(), "{report:?}");
6887 assert_eq!(report.evicted_copies, usize::from(cycle == 1 && pass == 2));
6888 assert_eq!(review.exists(), cycle == 1 && pass < 2);
6889 let census = source_census(&f.home);
6890 assert_eq!(census["current"]["next"], expected_next);
6891 assert_eq!(census["current"]["completed_at"].is_null(), pass < 2);
6892 assert_eq!(census["current"]["number"], cycle);
6893 assert_eq!(
6894 census["current"]["runs"]["20260101-000000-0000"]["root"]["outcome"],
6895 "error"
6896 );
6897 assert_eq!(
6898 census["current"]["runs"]["20260101-000000-0001"]["root"]["detail"],
6899 "root absent and no eviction records; nothing to reclaim"
6900 );
6901 if pass < 2 {
6902 let next_id = format!("20260101-000000-{expected_next:04x}");
6903 assert_eq!(
6904 census["current"]["runs"][next_id]["root"]["outcome"],
6905 "not-yet-inspected"
6906 );
6907 }
6908 }
6909 }
6910 }
6911
6912 #[tokio::test]
6913 async fn source_eviction_scan_reaches_old_seats_despite_empty_roots_and_timestamp_churn() {
6914 let f = Fx::new().await;
6915 let mut state = inactive_source(&f, "ffff", true).await;
6916 let review = reviewer(&f, &state, &f.base).await;
6917 let mut history = Vec::new();
6920 for i in 0..130 {
6921 let mut old = state.clone();
6922 old.id = format!("20260101-000000-{i:04x}");
6923 old.candidates.clear();
6924 old.save_under(&f.home).unwrap();
6925 std::fs::create_dir_all(old.worktree_root()).unwrap();
6926 history.push(old);
6927 }
6928 state
6929 .save_under_at(
6930 &f.home,
6931 Timestamp::now() + jiff::SignedDuration::from_secs(10),
6932 )
6933 .unwrap();
6934 for pass in 0..3 {
6935 let mut report = Report::default();
6936 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6937 assert!(report.failures.is_empty(), "{report:?}");
6938 let census = source_census(&f.home);
6939 assert_eq!(
6940 census["current"]["next"].as_u64().unwrap(),
6941 ((pass + 1) * 64).min(131)
6942 );
6943 if pass < 2 {
6944 assert!(review.exists());
6945 assert!(census["current"]["completed_at"].is_null());
6946 assert_eq!(
6947 census["current"]["runs"][&state.id]["root"]["outcome"],
6948 "not-yet-inspected"
6949 );
6950 }
6951 for old in history.iter_mut().take(64) {
6954 old.updated_at = Timestamp::now();
6955 old.save_under(&f.home).unwrap();
6956 }
6957 if pass == 1 {
6958 let older = state.updated_at - jiff::SignedDuration::from_secs(100);
6965 for old in history.iter_mut().take(128) {
6966 if f.home.join("runs").join(&old.id).exists() {
6967 old.save_under_at(&f.home, older).unwrap();
6968 }
6969 }
6970 history[127]
6971 .save_under_at(&f.home, older - jiff::SignedDuration::from_secs(100))
6972 .unwrap();
6973 let mut legacy: Vec<_> = history
6974 .iter()
6975 .filter(|r| f.home.join("runs").join(&r.id).exists())
6976 .cloned()
6977 .collect();
6978 legacy.push(state.clone());
6979 legacy.sort_by_key(|r| r.updated_at);
6980 let index = legacy.iter().position(|r| r.id == history[127].id).unwrap();
6981 let len = legacy.len();
6982 legacy.rotate_left((index + 1) % len);
6983 assert!(legacy.iter().take(64).all(|r| r.id != state.id));
6984 }
6985 if pass == 0 {
6986 std::fs::remove_dir_all(f.home.join("runs").join(&history[64].id)).unwrap();
6987 let mut added = state.clone();
6988 added.id = "20260101-000000-fffe".into();
6989 added.candidates.clear();
6990 added.save_under(&f.home).unwrap();
6991 }
6992 }
6993 assert!(!review.exists());
6994 let census = source_census(&f.home);
6995 assert!(!census["current"]["completed_at"].is_null());
6996 assert_eq!(
6997 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
6998 "evicted"
6999 );
7000 assert_eq!(
7001 census["current"]["runs"][&history[64].id]["root"]["outcome"],
7002 "error"
7003 );
7004 let mut next = Report::default();
7005 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
7006 let wrapped = source_census(&f.home);
7007 assert_eq!(wrapped["current"]["number"], 2);
7008 assert_eq!(wrapped["previous"], census["current"]);
7009 assert!(
7010 wrapped["current"]["runs"]
7011 .get("20260101-000000-fffe")
7012 .is_some()
7013 );
7014 }
7015
7016 #[tokio::test]
7017 async fn source_eviction_scan_resumes_within_run_and_bounds_failed_seats() {
7018 fn fail_one(path: &Path) -> Result<()> {
7019 ensure!(
7020 path.file_name().unwrap() != "review-2",
7021 "injected checkpoint failure"
7022 );
7023 Ok(())
7024 }
7025 let f = Fx::new().await;
7026 let state = inactive_source(&f, "ef20", true).await;
7027 let mut paths = Vec::new();
7028 for i in 1..=8 {
7029 let path = state.worktree_root().join(format!("review-{i}"));
7030 git::worktree_add_detached(&f.repo, &path, &f.base)
7031 .await
7032 .unwrap();
7033 paths.push(path);
7034 }
7035 std::fs::write(paths[2].join("owner-data"), "retain me").unwrap();
7036 let mut first = Report::default();
7037 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut first, fail_one).await;
7038 assert_eq!(first.evicted_copies, 3, "{first:?}");
7039 let census = source_census(&f.home);
7040 let seats = &census["current"]["runs"][&state.id]["seats"];
7041 assert_eq!(census["current"]["after_seat"], "review-5");
7042 assert_eq!(seats["review-2"]["outcome"], "error");
7043 assert_eq!(seats["review-2"]["stage"], "unlink");
7044 assert_eq!(seats["review-3"]["outcome"], "protected");
7045 assert_eq!(seats["review-6"]["outcome"], "not-yet-inspected");
7046 assert!(first.source_unlinked_logical_bytes > 0);
7047 assert!(first.source_evidence_added_logical_bytes > 0);
7048 let mut second = Report::default();
7049 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
7050 assert_eq!(second.evicted_copies, 3, "{second:?}");
7051 assert!(paths[1].exists(), "failed seat waits until next cycle");
7052 assert!(paths[2].exists());
7053 assert!(!source_census(&f.home)["current"]["completed_at"].is_null());
7054 let mut third = Report::default();
7055 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut third).await;
7056 assert_eq!(third.evicted_copies, 1, "{third:?}");
7057 assert!(!paths[1].exists());
7058 assert_eq!(
7059 std::fs::read_to_string(paths[2].join("owner-data")).unwrap(),
7060 "retain me"
7061 );
7062 let shown = seat_inspections(&f.home, &state.id).unwrap().join("\n");
7063 assert!(
7064 shown.contains("cycle=1 complete=true review-2 error stage=unlink"),
7065 "{shown}"
7066 );
7067 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
7068 let destination = f.bay.join("restored-scan-seat");
7069 restore_seat(&f.home, &state.id, &f.base, &destination)
7070 .await
7071 .unwrap();
7072 assert_eq!(git::rev_parse(&destination, "HEAD").await.unwrap(), f.base);
7073 }
7074
7075 #[tokio::test]
7076 async fn source_eviction_scan_continues_after_archive_failure_and_busy_output() {
7077 let f = Fx::new().await;
7078 let failed = inactive_source(&f, "ef30", true).await;
7079 let failed_seat = reviewer(&f, &failed, &f.base).await;
7080 let evidence = f
7081 .home
7082 .join("runs")
7083 .join(&failed.id)
7084 .join("artifacts/source-evidence");
7085 std::fs::create_dir_all(&evidence).unwrap();
7086 std::fs::File::create(evidence.join("reserved-evidence"))
7087 .unwrap()
7088 .set_len(64 * 1024 * 1024)
7089 .unwrap();
7090 let state = inactive_source(&f, "ef31", true).await;
7091 let busy = reviewer(&f, &state, &f.base).await;
7092 let available = state.worktree_root().join("review-2");
7093 git::worktree_add_detached(&f.repo, &available, &f.base)
7094 .await
7095 .unwrap();
7096 let output = OutputLease::take(&busy, true).unwrap();
7097 let mut report = Report::default();
7098 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7099 assert_eq!(report.evicted_copies, 1, "{report:?}");
7100 assert!(failed_seat.exists());
7101 assert!(busy.exists());
7102 assert!(!available.exists());
7103 let census = source_census(&f.home);
7104 assert!(!census["current"]["completed_at"].is_null());
7105 assert_eq!(
7106 census["current"]["runs"][&failed.id]["seats"]["review-1"]["outcome"],
7107 "error"
7108 );
7109 assert_eq!(
7110 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
7111 "protected"
7112 );
7113 drop(output);
7114 let mut retry = Report::default();
7115 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7116 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
7117 assert!(!busy.exists());
7118 assert!(failed_seat.exists());
7119 }
7120
7121 #[tokio::test]
7122 async fn source_eviction_scan_checkpoint_failure_stops_before_later_seats() {
7123 fn block_checkpoint(path: &Path) -> Result<()> {
7124 let home = path
7125 .parent()
7126 .unwrap()
7127 .parent()
7128 .unwrap()
7129 .parent()
7130 .unwrap()
7131 .join("home");
7132 let cursor = home.join("source-eviction-cursor");
7133 std::fs::rename(&cursor, home.join("saved-cursor"))?;
7134 std::fs::create_dir(&cursor)?;
7135 Ok(())
7136 }
7137 let f = Fx::new().await;
7138 let state = inactive_source(&f, "ef32", true).await;
7139 let first = reviewer(&f, &state, &f.base).await;
7140 let second = state.worktree_root().join("review-2");
7141 git::worktree_add_detached(&f.repo, &second, &f.base)
7142 .await
7143 .unwrap();
7144 let mut report = Report::default();
7145 copies::evict_with(
7146 &f.home,
7147 &Fx::disk(),
7148 Fx::now(),
7149 &mut report,
7150 block_checkpoint,
7151 )
7152 .await;
7153 assert_eq!(report.evicted_copies, 1, "{report:?}");
7154 assert!(!first.exists());
7155 assert!(second.exists());
7156 assert!(
7157 report
7158 .failures
7159 .iter()
7160 .any(|e| e.contains("persist source scan census"))
7161 );
7162 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
7163 std::fs::remove_dir(f.home.join("source-eviction-cursor")).unwrap();
7164 std::fs::rename(
7165 f.home.join("saved-cursor"),
7166 f.home.join("source-eviction-cursor"),
7167 )
7168 .unwrap();
7169 let mut retry = Report::default();
7170 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7171 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
7172 assert!(!second.exists());
7173 }
7174
7175 #[tokio::test]
7176 async fn source_eviction_scan_reports_invalid_cursor_and_global_ownership() {
7177 let f = Fx::new().await;
7178 let state = inactive_source(&f, "ef21", true).await;
7179 let review = reviewer(&f, &state, &f.base).await;
7180 let cursor = f.home.join("source-eviction-cursor");
7181 std::fs::write(&cursor, "broken cursor").unwrap();
7182 let mut report = Report::default();
7183 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7184 assert_eq!(std::fs::read_to_string(&cursor).unwrap(), "broken cursor");
7185 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
7186 assert!(review.exists());
7187 std::fs::write(&cursor, "20260101-000000-abcd").unwrap();
7189 let mut disabled = Fx::disk();
7190 disabled.auto_fold = false;
7191 let mut report = Report::default();
7192 copies::evict(&f.home, &disabled, Fx::now(), &mut report).await;
7193 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "disabled");
7194 let record = f.home.join("runs").join(&state.id).join("run.json");
7195 let original = std::fs::read(&record).unwrap();
7196 std::fs::write(&record, "invalid ownership").unwrap();
7197 let mut report = Report::default();
7198 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7199 assert!(
7200 report
7201 .failures
7202 .iter()
7203 .any(|e| e.contains("global run ownership validation"))
7204 );
7205 assert!(review.exists());
7206 std::fs::write(record, original).unwrap();
7207 let mut report = Report::default();
7208 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7209 assert_eq!(report.evicted_copies, 1, "{report:?}");
7210 assert_eq!(source_census(&f.home)["version"], 1);
7211 }
7212
7213 #[tokio::test]
7214 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
7215 let f = Fx::new().await;
7216 let mut runs = Vec::new();
7217 let mut paths = Vec::new();
7218 for tag in ["ec10", "ec11", "ec12", "ec13"] {
7219 let state = inactive_source(&f, tag, true).await;
7220 for name in ["judge-1", "advisor-1"] {
7221 let path = state.worktree_root().join(name);
7222 git::worktree_add_detached(&f.repo, &path, &f.base)
7223 .await
7224 .unwrap();
7225 paths.push(path);
7226 }
7227 runs.push(state);
7228 }
7229 let task = f.task(&runs);
7230 let mut report = Report::default();
7231 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7232 assert_eq!(report.evicted_copies, 5, "{report:?}");
7233 let mut next = Report::default();
7234 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
7235 assert_eq!(next.evicted_copies, 3, "{next:?}");
7236 assert!(paths.iter().all(|p| !p.exists()));
7237 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
7238 assert_eq!(
7239 crate::queue::Queue::at(f.home.join("queue"))
7240 .get(&task.id)
7241 .unwrap()
7242 .runs,
7243 task.runs
7244 );
7245 let limited = Fx::new().await;
7246 let state = inactive_source(&limited, "ec15", true).await;
7247 let copy = reviewer(&limited, &state, &limited.base).await;
7248 let evidence = limited
7249 .home
7250 .join("runs")
7251 .join(&state.id)
7252 .join("artifacts/source-evidence");
7253 std::fs::create_dir_all(&evidence).unwrap();
7254 let retained = evidence.join("reserved-evidence");
7255 std::fs::File::create(&retained)
7256 .unwrap()
7257 .set_len(64 * 1024 * 1024)
7258 .unwrap();
7259 let mut blocked = Report::default();
7260 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
7261 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
7262 assert!(copy.exists());
7263 assert_eq!(
7264 std::fs::metadata(&retained).unwrap().len(),
7265 64 * 1024 * 1024
7266 );
7267 assert!(
7268 blocked
7269 .retained
7270 .iter()
7271 .any(|r| r.contains("per-run storage limit")),
7272 "{blocked:?}"
7273 );
7274 }
7275 #[tokio::test]
7276 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
7277 let f = Fx::new().await;
7278 let mut state = inactive_source(&f, "ec14", true).await;
7279 historical_review(&mut state, &f.base);
7280 historical_review(&mut state, &f.base);
7281 state.status = RunStatus::Superseded;
7282 state.save_under(&f.home).unwrap();
7283 let review = reviewer(&f, &state, &f.base).await;
7284 let task = f.task(std::slice::from_ref(&state));
7285 let path = f.home.join("runs").join(&state.id).join("run.json");
7286 let mut raw: serde_json::Value =
7287 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
7288 raw["schema"] = 10.into();
7289 raw.as_object_mut().unwrap().remove("output_activity_at");
7290 raw["resource_wait"] = serde_json::Value::Null;
7293 raw["resource_block"] = serde_json::Value::Null;
7294 raw["driver_claimed"] = serde_json::Value::Null;
7295 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
7296 let graph = raw["config"]["graph"].as_object_mut().unwrap();
7298 let count = graph.remove("implementers").unwrap();
7299 graph.insert("candidates".into(), count);
7300 let queue = crate::queue::Queue::at(f.home.join("queue"));
7301 let task_path = queue.path_of(&task.id);
7302 let mut task_raw: serde_json::Value =
7303 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
7304 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
7305 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
7306 std::fs::write(&task_path, &task_historical).unwrap();
7307 let historical = serde_json::to_vec_pretty(&raw).unwrap();
7308 std::fs::write(&path, &historical).unwrap();
7309 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
7310 assert_eq!(report.evicted_copies, 1, "{report:?}");
7311 assert!(!review.exists());
7312 assert_eq!(std::fs::read(&path).unwrap(), historical);
7313 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
7314 assert!(state.candidates[0].worktree.exists());
7315 assert_eq!(
7316 crate::queue::Queue::at(f.home.join("queue"))
7317 .get(&task.id)
7318 .unwrap()
7319 .runs,
7320 task.runs
7321 );
7322 }
7323 #[tokio::test]
7324 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
7325 let f = Fx::new().await;
7326 let state = inactive_source(&f, "ed01", true).await;
7327 let review = reviewer(&f, &state, &f.base).await;
7328 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
7329 git::git(&review, &["add", "."]).await.unwrap();
7330 let tree = git::git(&review, &["write-tree"]).await.unwrap();
7331 let tip = git::git(
7332 &review,
7333 &[
7334 "commit-tree",
7335 &tree,
7336 "-p",
7337 &f.base,
7338 "-m",
7339 "temporary detached source",
7340 ],
7341 )
7342 .await
7343 .unwrap();
7344 git::git(&review, &["checkout", "--detach", &tip])
7345 .await
7346 .unwrap();
7347 git::git(&review, &["checkout", "--detach", &f.base])
7348 .await
7349 .unwrap();
7350 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
7351 .await
7352 .unwrap();
7353 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
7354 .await
7355 .unwrap();
7356 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
7357 .await
7358 .unwrap();
7359 assert!(!shown.lines().any(|oid| oid == tip));
7360 let admin = PathBuf::from(
7361 git::git(&review, &["rev-parse", "--absolute-git-dir"])
7362 .await
7363 .unwrap(),
7364 );
7365 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
7366 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
7367 fn interrupt(_: &Path) -> Result<()> {
7368 anyhow::bail!("fixture interruption before unlink")
7369 }
7370 let mut pending = Report::default();
7371 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
7372 assert!(review.exists());
7373 let ledger = f
7374 .home
7375 .join("runs")
7376 .join(&state.id)
7377 .join("artifacts/eviction-review-1.json");
7378 let complete = std::fs::read(&ledger).unwrap();
7379 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
7380 incomplete["history"]
7381 .as_object_mut()
7382 .unwrap()
7383 .remove(&tip)
7384 .unwrap();
7385 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
7386 let mut blocked = Report::default();
7387 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
7388 assert!(review.exists());
7389 assert!(
7390 blocked
7391 .retained
7392 .iter()
7393 .any(|reason| reason.contains("incomplete reflog checkpoint")),
7394 "{blocked:?}"
7395 );
7396 std::fs::write(&ledger, complete).unwrap();
7397 let mut report = Report::default();
7398 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7399 assert_eq!(report.evicted_copies, 1, "{report:?}");
7400 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
7401 .await
7402 .unwrap();
7403 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
7404 assert!(
7405 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
7406 .await
7407 .unwrap()
7408 .ok()
7409 );
7410 std::fs::remove_dir_all(&f.repo).unwrap();
7411 let destination = f._temp.path().join("restore-old-only");
7412 restore_seat(&f.home, &state.id, &tip, &destination)
7413 .await
7414 .unwrap();
7415 assert_eq!(
7416 std::fs::read(destination.join("old-only")).unwrap(),
7417 b"old-only\0binary\xff"
7418 );
7419 }
7420
7421 #[tokio::test]
7422 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
7423 fn interrupt(_: &Path) -> Result<()> {
7424 anyhow::bail!("fixture interruption before unlink")
7425 }
7426 fn vanish(path: &Path) -> Result<()> {
7427 std::fs::remove_dir_all(path)?;
7428 anyhow::bail!("fixture interruption before unregistration")
7429 }
7430 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
7431 let f = Fx::new().await;
7432 let mut state = inactive_source(&f, "ed02", true).await;
7433 state.driver_exited = false;
7434 state.driver_claimed = Some(true);
7435 state.driver_pid = Some(std::process::id());
7436 state.driver_started_at = crate::proc::process_started_at(std::process::id());
7437 state.begin_pass();
7438 state.status = RunStatus::Superseded;
7439 state.save_under(&f.home).unwrap();
7440 let review = reviewer(&f, &state, &f.base).await;
7441 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
7444 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
7445 std::fs::write(
7446 f.home.join("runs").join(&state.id).join("driver.json"),
7447 serde_json::to_vec(&identity).unwrap(),
7448 )
7449 .unwrap();
7450 let live = if stage == "live-driver" {
7451 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
7452 } else {
7453 None
7454 };
7455 assert_eq!(
7456 state.liveness_under(&f.home, false),
7457 if live.is_some() {
7458 Liveness::Live
7459 } else {
7460 Liveness::Dead
7461 }
7462 );
7463 let mut report = Report::default();
7464 if stage == "intent" || stage == "missing-registration" {
7465 copies::evict_with(
7466 &f.home,
7467 &Fx::disk(),
7468 Fx::now(),
7469 &mut report,
7470 if stage == "intent" { interrupt } else { vanish },
7471 )
7472 .await;
7473 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
7474 let mut retry = Report::default();
7475 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7476 assert!(!review.exists(), "{stage}: {retry:?}");
7477 let ledger = f
7478 .home
7479 .join("runs")
7480 .join(&state.id)
7481 .join("artifacts/eviction-review-1.json");
7482 let receipt: serde_json::Value =
7483 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
7484 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
7485 } else {
7486 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7487 assert_eq!(
7488 report.evicted_copies,
7489 usize::from(live.is_none()),
7490 "{stage}: {report:?}"
7491 );
7492 assert_eq!(review.exists(), live.is_some());
7493 }
7494 drop(live);
7495 assert!(
7496 !RunState::load_under(&state.id, &f.home)
7497 .unwrap()
7498 .driver_exited
7499 );
7500 }
7501 }
7502}