1mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7 let artifacts = home.join("runs").join(&state.id).join("artifacts");
8 if !artifacts.try_exists()? {
9 return Ok(false);
10 }
11 ensure!(
12 plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13 "noncanonical artifacts"
14 );
15 for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16 ensure!(i < 4096, "eviction record inspection limit exceeded");
17 let name = entry?.file_name();
18 let name = name.to_string_lossy();
19 if name.starts_with("eviction-") && name.ends_with(".json") {
20 return Ok(true);
21 }
22 }
23 Ok(false)
24}
25
26pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29 copies::restore_execution_seats(home, state).await
30}
31
32pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34 crate::fold_evidence::validate_snapshot(evidence).await?;
35 Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40 crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
45 copies::seat_evidence(home, id).await
46}
47
48pub async fn seat_heads(
50 home: &Path,
51 id: &str,
52) -> Result<std::collections::BTreeMap<String, String>> {
53 copies::seat_heads(home, id).await
54}
55
56pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
58 copies::seat_diff(home, id, head).await
59}
60
61pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
63 copies::restore_seat(home, id, head, destination).await
64}
65
66use crate::{
67 git,
68 proc::Quiet as _,
69 run::{Liveness, RunState},
70};
71use anyhow::{Context, Result, ensure};
72use jiff::Timestamp;
73use serde::{Deserialize, Serialize};
74use std::{
75 fs::{File, OpenOptions},
76 path::{Path, PathBuf},
77};
78
79#[derive(Debug)]
83pub(crate) struct OutputLease {
84 _file: File,
85 _activity: bool,
86}
87
88impl OutputLease {
89 #[cfg(unix)]
90 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
91 use std::os::fd::AsRawFd;
92 self._file.as_raw_fd()
93 }
94
95 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
96 if !worktree.join(".git").is_file() {
97 return Ok(None);
98 }
99 Self::take(worktree, false).map(Some)
100 }
101 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
102 use std::hash::{Hash, Hasher};
103 let mut hash = std::collections::hash_map::DefaultHasher::new();
104 let mut normalized = cache.to_owned();
105 let mut missing = Vec::new();
106 while !normalized.exists() {
107 let Some(name) = normalized.file_name() else {
108 break;
109 };
110 missing.push(name.to_owned());
111 if !normalized.pop() {
112 break;
113 }
114 }
115 normalized = normalized.canonicalize().unwrap_or(normalized);
116 for part in missing.into_iter().rev() {
117 normalized.push(part);
118 }
119 normalized.hash(&mut hash);
120 let dir = home.join("cache-leases");
121 std::fs::create_dir_all(&dir)?;
122 Self::lock(
123 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
124 exclusive,
125 )
126 }
127 #[cfg(unix)]
131 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
132 Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
133 }
134
135 #[cfg(unix)]
136 pub(crate) fn admit_with(
137 home: &Path,
138 paths: &[PathBuf],
139 min_free: u64,
140 free_bytes: impl Fn(&Path) -> Result<u64>,
141 ) -> Result<Vec<Self>> {
142 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
143 if min_free == 0 {
144 return Ok(Vec::new());
145 }
146 const RESERVE: u64 = 1024 * 1024 * 1024;
147 std::fs::create_dir_all(home)?;
148 let dir = home.canonicalize()?.join("disk-admission");
149 std::fs::create_dir_all(&dir)?;
150 ensure!(
151 plain_dir(&dir) && dir.canonicalize()? == dir,
152 "disk admission directory is not canonical"
153 );
154 let mut devices = std::collections::BTreeSet::new();
155 let mut reservations = Vec::new();
156 for path in paths {
157 let probe = path
158 .ancestors()
159 .find(|p| p.exists())
160 .context("disk admission path missing")?;
161 let device = std::fs::metadata(probe)?.dev();
162 if !devices.insert(device) {
163 continue;
164 }
165 let open = |path: &Path| -> Result<File> {
166 Ok(OpenOptions::new()
167 .read(true)
168 .write(true)
169 .create(true)
170 .truncate(false)
171 .custom_flags(libc::O_NOFOLLOW)
172 .open(path)?)
173 };
174 let admission = Self {
178 _file: open(&dir.join(format!("{device}.lock")))?,
179 _activity: false,
180 };
181 let started = std::time::Instant::now();
182 loop {
183 match crate::pr_head::try_lock_file(&admission._file) {
184 Ok(()) => break,
185 Err(error)
186 if matches!(
187 error.kind(),
188 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
189 ) && started.elapsed() < std::time::Duration::from_secs(1) =>
190 {
191 std::thread::sleep(std::time::Duration::from_millis(5));
192 }
193 Err(error) => {
194 return Err(error)
195 .context("disk admission busy or unavailable; retry without dispatch");
196 }
197 }
198 }
199 let mut active = 0u64;
200 let mut available = None;
201 for slot in 0..1024 {
202 let file = dir.join(format!("{device}-{slot}.reserve"));
203 if !file.exists() {
204 if available.is_none() {
205 available = Some(file);
206 }
207 continue;
208 }
209 let file = open(&file)?;
210 match crate::pr_head::try_lock_file(&file) {
211 Ok(()) if available.is_none() => {
212 available = Some(dir.join(format!("{device}-{slot}.reserve")))
213 }
214 Ok(()) => {}
215 Err(_) => active = active.saturating_add(RESERVE),
216 }
217 }
218 let free = free_bytes(probe)?;
219 let required = min_free.saturating_add(active).saturating_add(RESERVE);
220 ensure!(
221 free >= required,
222 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
223 );
224 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
225 crate::pr_head::try_lock_file(&file)?;
226 reservations.push(Self {
227 _file: file,
228 _activity: false,
229 });
230 }
231 Ok(reservations)
232 }
233
234 #[cfg(not(unix))]
235 pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
236 Ok(Vec::new())
238 }
239
240 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
241 let out = std::process::Command::new("git")
242 .args(["rev-parse", "--absolute-git-dir"])
243 .current_dir(worktree)
244 .quiet()
245 .output()?;
246 ensure!(
247 out.status.success(),
248 "cannot identify output lease directory"
249 );
250 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
251 Self::lock(&dir.join("magi-output.lock"), exclusive)
252 }
253 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
254 let file = OpenOptions::new()
255 .create(true)
256 .truncate(false)
257 .read(true)
258 .write(true)
259 .open(path)
260 .with_context(|| format!("output lease: open {}", path.display()))?;
261 #[cfg(unix)]
262 {
263 use std::os::fd::AsRawFd;
264 let mode = if exclusive {
265 libc::LOCK_EX
266 } else {
267 libc::LOCK_SH
268 };
269 ensure!(
271 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
272 "output lease: outputs are in use or being reclaimed: {}",
273 path.display()
274 );
275 }
276 #[cfg(not(unix))]
277 let _ = exclusive;
278 if !exclusive {
279 file.set_len(1)?;
281 file.set_modified(std::time::SystemTime::now())
282 .with_context(|| format!("record output activity {}", path.display()))?;
283 }
284 Ok(Self {
285 _file: file,
286 _activity: !exclusive,
287 })
288 }
289}
290#[cfg(unix)]
291impl Drop for OutputLease {
292 fn drop(&mut self) {
293 use std::os::fd::AsRawFd;
294 if self._activity
295 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
296 {
297 let _ = self._file.set_len(2);
300 tracing::warn!("could not record output lease release activity: {error}");
301 }
302 unsafe {
304 libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
305 }
306 }
307}
308
309#[derive(Default, Debug, Serialize, Deserialize)]
310#[serde(default)]
311pub(crate) struct Report {
312 pub checked_at: Option<Timestamp>,
313 pub limit_bytes: u64,
314 pub reclaim_budget_bytes: u64,
315 pub measured_bytes: u64,
317 pub freed_bytes: u64,
319 pub measured_allocated_file_bytes: Option<u64>,
320 pub proven_reclaimable_bytes: u64,
322 pub proven_reclaimable_allocated_bytes: Option<u64>,
325 pub protected_or_unproven_bytes: u64,
326 pub estimated_released_allocated_bytes: Option<u64>,
329 pub observed_home_free_change_bytes: Option<i64>,
332 pub retired: usize,
333 pub evicted_copies: usize,
334 pub removed_empty_roots: usize,
335 pub source_unlinked_logical_bytes: u64,
336 pub source_unlinked_allocated_bytes: Option<u64>,
337 pub source_evidence_added_logical_bytes: u64,
338 pub source_evidence_added_allocated_bytes: Option<u64>,
339 pub retained: Vec<String>,
340 pub failures: Vec<String>,
341}
342
343#[cfg(unix)]
344fn allocated_inventory(
345 root: &Path,
346 seen: &mut std::collections::BTreeSet<(u64, u64)>,
347) -> Result<u64> {
348 use std::os::unix::fs::MetadataExt;
349 ensure!(
350 plain_dir(root) && root.canonicalize()? == root,
351 "allocated inventory root is not canonical"
352 );
353 let mut dirs = vec![root.to_owned()];
354 let mut bytes = 0u64;
355 while let Some(dir) = dirs.pop() {
356 for entry in std::fs::read_dir(dir)? {
357 let path = entry?.path();
358 let metadata = std::fs::symlink_metadata(&path)?;
359 if metadata.is_dir() {
360 dirs.push(path);
361 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
362 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
363 }
364 }
365 }
366 Ok(bytes)
367}
368
369#[derive(Default)]
370struct ReleasedAllocation {
371 #[cfg(unix)]
372 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
373 #[cfg(unix)]
374 removed: std::collections::BTreeMap<(u64, u64), u64>,
375}
376
377impl ReleasedAllocation {
378 fn of(paths: &[PathBuf]) -> Result<Self> {
379 #[cfg(unix)]
380 {
381 use std::os::unix::fs::MetadataExt;
382 let mut files = std::collections::BTreeMap::new();
383 for path in paths {
384 let m = std::fs::symlink_metadata(path)?;
385 ensure!(m.is_file(), "allocation candidate is not regular");
386 files.insert(
387 path.clone(),
388 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
389 );
390 }
391 Ok(Self {
392 files,
393 removed: Default::default(),
394 })
395 }
396 #[cfg(not(unix))]
397 {
398 let _ = paths;
399 Ok(Self::default())
400 }
401 }
402 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
403 #[cfg(unix)]
404 {
405 let mut inodes = std::collections::BTreeMap::new();
406 for &(device, inode, links, blocks) in self.files.values() {
407 let entry = inodes
408 .entry((device, inode))
409 .or_insert((0u64, links, blocks));
410 entry.0 += 1;
411 }
412 Some(
413 inodes
414 .values()
415 .filter(|(count, links, _)| count == links)
416 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
417 )
418 }
419 #[cfg(not(unix))]
420 {
421 None
422 }
423 }
424
425 fn unlinked(&mut self, path: &Path) -> Option<u64> {
426 #[cfg(unix)]
427 {
428 let (device, inode, links, blocks) = self.files[path];
429 let removed = self.removed.entry((device, inode)).or_default();
430 *removed += 1;
431 Some(if *removed == links { blocks } else { 0 })
432 }
433 #[cfg(not(unix))]
434 {
435 let _ = path;
436 None
437 }
438 }
439}
440
441fn plain_dir(path: &Path) -> bool {
442 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
443}
444
445fn run_paths_missing(state: &RunState) -> Result<bool> {
448 for path in [state.worktree_root(), state.repo.clone()] {
449 match std::fs::metadata(&path) {
450 Ok(_) => {}
451 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
452 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
453 }
454 }
455 Ok(false)
456}
457
458async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
460 let root = state.worktree_root();
461 if !plain_dir(&root) {
462 return Ok(Vec::new());
463 }
464 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
465 registered_worktrees(state, &list)
466}
467
468fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
469 let root = state.worktree_root();
470 if !plain_dir(&root) {
471 return Ok(Vec::new());
472 }
473 let root = root.canonicalize()?;
474 let mut paths = Vec::new();
475 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
476 let path = PathBuf::from(path);
477 #[cfg(windows)]
478 let path = if plain_dir(&path) {
479 git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
480 } else {
481 continue;
482 };
483 if plain_dir(&path)
484 && path.parent() == Some(root.as_path())
485 && path.canonicalize()? == path
486 && path.join(".git").is_file()
487 {
488 paths.push(path);
489 }
490 }
491 Ok(paths)
492}
493
494async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
497 validate_cargo_target(worktree, false).await
498}
499
500async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
501 let target = worktree.join("target");
502 ensure!(plain_dir(&target), "target absent or not a plain directory");
503 ensure!(target.canonicalize()? == target, "target path changed");
504 ensure!(
505 git::git(worktree, &["ls-files", "--", "target"])
506 .await?
507 .is_empty(),
508 "tracked target contents"
509 );
510 ensure!(
511 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
512 .await?
513 .ok(),
514 "target is not ignored"
515 );
516 cache_tag(&target, allow_missing_tag)?;
517 let path = target.join(".rustc_info.json");
518 plain_file(&path, "rustc identity")?;
519 let info: serde_json::Value = serde_json::from_slice(
520 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
521 )
522 .with_context(|| format!("parse rustc identity {}", path.display()))?;
523 ensure!(
524 info.get("rustc_fingerprint").is_some(),
525 "unknown Cargo identity {}",
526 path.display()
527 );
528 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
529 ensure!(
530 info.get("rustc_fingerprint")
531 .is_some_and(serde_json::Value::is_u64)
532 && info
533 .get("outputs")
534 .and_then(serde_json::Value::as_object)
535 .is_some_and(|outputs| outputs.values().any(|output| {
536 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
537 && output
538 .get("stdout")
539 .and_then(serde_json::Value::as_str)
540 .is_some_and(|stdout| stdout.starts_with("rustc "))
541 && output
542 .get("stderr")
543 .is_some_and(serde_json::Value::is_string)
544 }))
545 && info
546 .get("successes")
547 .is_some_and(serde_json::Value::is_object),
548 "missing-tag rustc identity structure invalid: {}",
549 path.display()
550 );
551 let mut profiles = 0;
552 for profile in ["debug", "release"] {
553 let dir = target.join(profile);
554 if !dir.exists() {
555 continue;
556 }
557 for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
558 ensure!(
559 plain_dir(&subdir) && subdir.canonicalize()? == subdir,
560 "missing-tag profile proof: {} is not a plain directory",
561 subdir.display()
562 );
563 }
564 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
565 profiles += 1;
566 }
567 ensure!(
568 profiles > 0,
569 "missing-tag proof: no Cargo profile in {}",
570 target.display()
571 );
572 }
573 for name in ["debug", "release"] {
574 let path = target.join(name);
575 match std::fs::symlink_metadata(&path) {
576 Ok(metadata) => ensure!(
577 metadata.is_dir() && path.canonicalize()? == path,
578 "profile validation: not a plain directory: {}",
579 path.display()
580 ),
581 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
582 Err(error) => {
583 return Err(error)
584 .with_context(|| format!("profile validation: inspect {}", path.display()));
585 }
586 }
587 }
588 Ok(target)
589}
590
591fn plain_file(path: &Path, stage: &str) -> Result<()> {
592 let meta = std::fs::symlink_metadata(path)
593 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
594 ensure!(
595 meta.is_file() && path.canonicalize()? == path,
596 "{stage}: not a plain file: {}",
597 path.display()
598 );
599 Ok(())
600}
601
602fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
603 let path = target.join("CACHEDIR.TAG");
604 match std::fs::symlink_metadata(&path) {
605 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
606 return Ok(());
607 }
608 result => {
609 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
610 }
611 }
612 plain_file(&path, "cache tag")?;
613 let tag = std::fs::read_to_string(&path)
614 .with_context(|| format!("cache tag: read {}", path.display()))?;
615 ensure!(
616 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
617 "unknown Cargo cache tag: {}",
618 path.display()
619 );
620 Ok(())
621}
622
623fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
627 let mut dirs = vec![target.to_owned()];
628 let mut entries = 0;
629 while let Some(dir) = dirs.pop() {
630 for entry in std::fs::read_dir(&dir)
631 .with_context(|| format!("output activity: list {}", dir.display()))?
632 {
633 let path = entry?.path();
634 entries += 1;
635 ensure!(
636 entries <= 100_000,
637 "output activity inventory too large: {}",
638 target.display()
639 );
640 let metadata = std::fs::symlink_metadata(&path)
641 .with_context(|| format!("output activity: inspect {}", path.display()))?;
642 if metadata.is_dir() {
643 if dir != target
645 || matches!(
646 path.file_name().and_then(|s| s.to_str()),
647 Some("debug" | "release")
648 )
649 {
650 dirs.push(path);
651 }
652 } else if metadata.is_file()
653 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
654 {
655 let at =
656 Timestamp::try_from(metadata.modified().with_context(|| {
657 format!("output activity: modified {}", path.display())
658 })?)?;
659 ensure!(
660 crate::clean::due(now, at, grace),
661 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
662 path.display()
663 );
664 }
665 }
666 }
667 Ok(())
668}
669
670pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
671 use std::io::Read as _;
672 let mut header = [0; 8];
673 let n = File::open(path)?.read(&mut header)?;
674 let header = &header[..n];
675 Ok(header.starts_with(b"\x7fELF")
676 || header.starts_with(b"!<arch>\n")
677 || header.starts_with(b"rust\0\0\0")
678 || header.starts_with(b"MZ\x90\0")
679 || [
680 [0xfe, 0xed, 0xfa, 0xce],
681 [0xce, 0xfa, 0xed, 0xfe],
682 [0xfe, 0xed, 0xfa, 0xcf],
683 [0xcf, 0xfa, 0xed, 0xfe],
684 [0xca, 0xfe, 0xba, 0xbe],
685 [0xca, 0xfe, 0xba, 0xbf],
686 ]
687 .iter()
688 .any(|magic| header.starts_with(magic)))
689}
690
691fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
695 let mut outputs = std::collections::BTreeSet::new();
696 for profile in ["debug", "release"] {
697 for dir in [target.join(profile), target.join(profile).join("deps")] {
698 if !plain_dir(&dir) || dir.canonicalize()? != dir {
699 continue;
700 }
701 for entry in std::fs::read_dir(&dir)? {
702 let path = entry?.path();
703 if path.extension().is_none_or(|ext| ext != "d")
704 || !std::fs::symlink_metadata(&path)?.is_file()
705 || std::fs::metadata(&path)?.len() > 1024 * 1024
706 || path.canonicalize()? != path
707 {
708 continue;
709 }
710 let body = std::fs::read_to_string(&path)?;
711 for line in body.lines() {
712 let Some((names, dependencies)) = line.split_once(": ") else {
713 continue;
714 };
715 if dependencies.trim().is_empty() {
716 continue;
717 }
718 let mut words = Vec::new();
721 let mut word = String::new();
722 let mut escaped = false;
723 for c in names.chars() {
724 if escaped {
725 word.push(c);
726 escaped = false;
727 } else if c == '\\' {
728 escaped = true;
729 } else if c.is_whitespace() {
730 if !word.is_empty() {
731 words.push(std::mem::take(&mut word));
732 }
733 } else {
734 word.push(c);
735 }
736 }
737 if escaped || word.contains(['$', '%']) {
738 continue;
739 }
740 if !word.is_empty() {
741 words.push(word);
742 }
743 for name in words {
744 let output = PathBuf::from(name);
745 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
746 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
747 let artifact_name =
748 output_stem == stem || output_stem == format!("lib{stem}");
749 let artifact_kind = output.extension().is_none_or(|ext| {
750 matches!(
751 ext.to_str(),
752 Some(
753 "rlib"
754 | "rmeta"
755 | "so"
756 | "dylib"
757 | "a"
758 | "lib"
759 | "dll"
760 | "exe"
761 | "pdb"
762 )
763 )
764 });
765 if artifact_name
766 && artifact_kind
767 && output.parent() == Some(dir.as_path())
768 && output.extension().is_none_or(|ext| ext != "d")
769 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
770 && output.canonicalize().is_ok_and(|p| p == output)
771 && artifact_signature(&output)?
772 {
773 outputs.insert(output);
774 }
775 }
776 }
777 }
778 }
779 }
780 Ok(outputs.into_iter().collect())
781}
782
783#[derive(Serialize, Deserialize)]
784struct CargoResidue {
785 target: PathBuf,
786 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
787}
788
789fn residue_files(
792 target: &Path,
793 outputs: &[PathBuf],
794) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
795 let mut files = std::collections::BTreeMap::new();
796 let mut dirs = vec![target.to_owned()];
797 let mut bytes = 0usize;
798 let mut entries = 0usize;
799 while let Some(dir) = dirs.pop() {
800 for entry in std::fs::read_dir(dir)? {
801 let path = entry?.path();
802 entries += 1;
803 ensure!(entries <= 4096, "residue inventory too large");
804 let metadata = std::fs::symlink_metadata(&path)?;
805 ensure!(
806 !metadata.file_type().is_symlink(),
807 "symlink in Cargo residue"
808 );
809 if metadata.is_dir() {
810 dirs.push(path);
811 } else {
812 ensure!(metadata.is_file(), "non-file in Cargo residue");
813 if outputs.contains(&path) {
814 continue;
815 }
816 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
817 let body = std::fs::read(&path)?;
818 bytes = bytes.saturating_add(body.len());
819 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
820 files.insert(path.strip_prefix(target)?.to_owned(), body);
821 }
822 }
823 }
824 Ok(files)
825}
826
827fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
828 Ok(home
829 .join("runs")
830 .join(id)
831 .join("build-residue")
832 .join(worktree.file_name().context("missing worktree name")?))
833}
834
835fn residue_root(target: &Path) -> Result<()> {
838 for entry in std::fs::read_dir(target)? {
839 let name = entry?.file_name();
840 ensure!(
841 matches!(
842 name.to_str(),
843 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
844 ),
845 "unrecognized Cargo residue root entry"
846 );
847 }
848 Ok(())
849}
850
851fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
852 cache_tag(target, false)?;
853 residue_root(target)?;
854 let files = residue_files(target, outputs)?;
855 for (relative, body) in &files {
856 let path = target.join(relative);
857 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
858 let lock = matches!(
859 relative.to_str(),
860 Some("debug/.cargo-lock" | "release/.cargo-lock")
861 ) && body.is_empty();
862 let dep_info = path.extension().is_some_and(|ext| ext == "d")
863 && outputs.iter().any(|output| {
864 output.parent() == path.parent()
865 && (output.file_stem() == path.file_stem()
866 || output.file_stem().is_some_and(|stem| {
867 stem.to_string_lossy()
868 == format!(
869 "lib{}",
870 path.file_stem().unwrap_or_default().to_string_lossy()
871 )
872 }))
873 });
874 ensure!(identity || lock || dep_info, "unproven Cargo residue");
875 }
876 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
877 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
878 let tmp = proof.with_extension("tmp");
879 std::fs::write(
880 &tmp,
881 serde_json::to_vec(&CargoResidue {
882 target: target.to_owned(),
883 files,
884 })?,
885 )?;
886 std::fs::rename(tmp, proof)?;
887 Ok(())
888}
889
890pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
891 let target = cargo_target(worktree).await?;
892 residue_root(&target)?;
893 let locks = cargo_locks(&target)?;
894 let proof: CargoResidue =
895 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
896 ensure!(
897 proof.target == target && proof.files == residue_files(&target, &[])?,
898 "Cargo residue changed or contains unproven files"
899 );
900 Ok(locks)
901}
902
903pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
905 let mut held = Vec::new();
906 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
907 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
908 }) {
909 let lock = target.join(profile).join(name);
910 match std::fs::symlink_metadata(&lock) {
911 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
912 result => {
913 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
914 }
915 }
916 {
917 plain_file(&lock, "Cargo lock")?;
918 let file = OpenOptions::new()
919 .read(true)
920 .write(true)
921 .open(&lock)
922 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
923 crate::pr_head::try_lock_file(&file)
924 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
925 held.push(file);
926 }
927 }
928 Ok(held)
929}
930
931pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
934 let home = home.to_owned();
935 let disk = disk.clone();
936 let runtime = tokio::runtime::Handle::current();
937 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
938 .await
939 .unwrap_or_else(|e| Report {
940 failures: vec![format!("retention worker failed: {e}")],
941 ..Report::default()
942 })
943}
944
945#[derive(Default, Serialize, Deserialize)]
946struct Measurement {
947 at: Option<Timestamp>,
948 bytes: u64,
949}
950
951async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
952 let mut report = Report {
953 checked_at: Some(now),
954 limit_bytes: disk.worktree_limit_bytes,
955 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
956 ..Report::default()
957 };
958 if !cfg!(unix) {
959 report
960 .failures
961 .push("Automatic output reclamation requires Unix process containment".into());
962 return report;
963 }
964 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
965 return report;
966 };
967 let mut targets = Vec::new();
968 let mut under_pressure = false;
969 let mut registrations = std::collections::HashMap::new();
970 let measurements_path = home.join("output-measurements.json");
971 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
972 std::fs::read(&measurements_path)
973 .ok()
974 .and_then(|b| serde_json::from_slice(&b).ok())
975 .unwrap_or_default();
976 let mut observed = std::collections::BTreeSet::new();
977 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
978 .into_iter()
979 .flatten()
980 .flatten()
981 .map(|e| e.file_name().to_string_lossy().into_owned())
982 .collect();
983 ids.sort();
984 for id in ids {
985 if !crate::run::is_run_id(&id) {
986 continue;
987 }
988 let state = match RunState::load_under(&id, home) {
989 Ok(state) => state,
990 Err(e) => {
991 report
992 .retained
993 .push(format!("{id}: unreadable run ownership: {e:#}"));
994 continue;
995 }
996 };
997 if disk.worktree_limit_bytes == 0 {
998 continue;
999 }
1000 match run_paths_missing(&state) {
1001 Ok(true) => continue,
1002 Ok(false) => {}
1003 Err(e) => {
1004 report.failures.push(format!("{id}: {e:#}"));
1005 continue;
1006 }
1007 }
1008 let list = match registrations.entry(state.repo.clone()) {
1009 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1010 std::collections::hash_map::Entry::Vacant(entry) => {
1011 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1012 Ok(list) => entry.insert(list),
1013 Err(e) => {
1014 if matches!(run_paths_missing(&state), Ok(true)) {
1017 continue;
1018 }
1019 report.failures.push(format!("{id}: {e:#}"));
1020 continue;
1021 }
1022 }
1023 }
1024 };
1025 let paths = match registered_worktrees(&state, list) {
1026 Ok(paths) => paths,
1027 Err(e) => {
1028 report.failures.push(format!("{id}: {e:#}"));
1029 continue;
1030 }
1031 };
1032 for path in paths {
1033 if !path.join("target").exists() {
1034 continue;
1035 }
1036 let target = path.join("target");
1037 observed.insert(target.clone());
1038 let pressure =
1039 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1040 under_pressure |= pressure;
1041 let measurement = measurements.entry(target).or_default();
1042 if pressure
1043 || measurement
1044 .at
1045 .is_none_or(|at| crate::clean::due(now, at, 300))
1046 {
1047 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1048 measurement.at = Some(now);
1049 }
1050 let size = measurement.bytes;
1051 report.measured_bytes = report.measured_bytes.saturating_add(size);
1052 targets.push((state.output_activity(), id.clone(), path, size));
1053 }
1054 }
1055 if under_pressure {
1056 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1059 } else {
1060 targets.sort_by_key(|t| t.0);
1061 }
1062 let free_before = crate::disk::free_bytes(home).ok();
1063 #[cfg(unix)]
1064 {
1065 let mut seen = std::collections::BTreeSet::new();
1066 report.measured_allocated_file_bytes = Some(0);
1067 report.estimated_released_allocated_bytes = Some(0);
1068 report.proven_reclaimable_allocated_bytes = Some(0);
1069 for (_, _, path, _) in &targets {
1070 report.measured_allocated_file_bytes = report
1071 .measured_allocated_file_bytes
1072 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1073 .map(|(a, b)| a.saturating_add(b));
1074 }
1075 }
1076 let mut remaining = report.measured_bytes;
1077 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1078 let mut deletion_started = false;
1079 let mut oversized_started = false;
1080 for (updated, id, path, _) in targets {
1081 if disk.worktree_limit_bytes == 0
1082 || remaining <= disk.worktree_limit_bytes
1083 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1084 {
1085 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1086 continue;
1087 }
1088 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1089 report
1090 .retained
1091 .push(format!("{id}: per-pass byte budget spent"));
1092 continue;
1093 }
1094 let result = async {
1095 ensure!(
1096 crate::clean::due(now, updated, disk.fold_grace_secs),
1097 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1098 );
1099 let _owner = crate::pr_head::owner_lock(home, &id)?;
1100 let state = RunState::load_under(&id, home)?;
1101 ensure!(
1102 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1103 == Liveness::Dead,
1104 "run live or ownership unknown"
1105 );
1106 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1107 let _driver = crate::driver::maintenance(home, &id)?;
1108 let _outputs = OutputLease::take(&path, true)?;
1109 ensure!(
1110 worktrees(&state).await?.contains(&path),
1111 "worktree registration changed"
1112 );
1113 let target = validate_cargo_target(&path, true).await
1114 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1115 let _cargo = cargo_locks(&target)
1116 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1117 validate_cargo_target(&path, true).await
1120 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1121 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1122 if lease_meta.len() > 0 {
1123 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1124 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1125 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1126 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1127 }
1128 output_age(&target, now, disk.fold_grace_secs)
1129 .with_context(|| format!("stage output-age: {}", target.display()))?;
1130 let mut outputs = declared_outputs(&target)
1131 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1132 let incremental = match crate::incremental::candidates(&path) {
1133 Ok(candidates) => candidates,
1134 Err(error) => {
1135 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1136 crate::incremental::Reclaim::default()
1137 }
1138 };
1139 let incremental_files = incremental.files.clone();
1140 outputs.extend(incremental_files.iter().cloned());
1141 outputs.sort();
1142 outputs.dedup();
1143 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1144 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1145 .with_context(|| format!("stage measure: {}", p.display())))
1146 .collect::<Result<Vec<_>>>()?;
1147 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1148 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1149 let mut allocation = ReleasedAllocation::of(&outputs)?;
1150 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1151 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1152 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1153 ensure!(!deletion_started || !exceeds_budget,
1157 "declared outputs exceed remaining per-pass byte budget");
1158 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1161 std::fs::write(
1162 &ledger,
1163 serde_json::to_vec(
1164 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1165 )?,
1166 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1167 if !incremental_files.is_empty() {
1170 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1171 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1172 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1173 }
1174 ensure!(worktrees(&state).await?.contains(&path),
1175 "worktree registration changed before deletion");
1176 deletion_started = true;
1177 oversized_started = exceeds_budget;
1178 measurements.remove(&target);
1179 for (output, size) in outputs.iter().zip(sizes) {
1180 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1181 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1182 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1183 report.freed_bytes = report.freed_bytes.saturating_add(size);
1185 remaining = remaining.saturating_sub(size);
1186 }
1187 incremental.finish().context("finish incremental reclamation")?;
1188 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1189 Ok::<_, anyhow::Error>(())
1190 }
1191 .await;
1192 if let Err(e) = result {
1193 report.retained.push(format!(
1194 "{id}/{}: {e:#}",
1195 path.file_name().unwrap_or_default().to_string_lossy()
1196 ));
1197 }
1198 }
1199 measurements.retain(|path, _| observed.contains(path));
1200 if let Ok(body) = serde_json::to_vec(&measurements)
1201 && let Err(e) = std::fs::write(&measurements_path, body)
1202 {
1203 report.failures.push(format!("output measurements: {e}"));
1204 }
1205 copies::evict(home, disk, now, &mut report).await;
1206 retire_empty(home, disk, now, &mut report).await;
1207 report.protected_or_unproven_bytes = report
1208 .measured_bytes
1209 .saturating_sub(report.proven_reclaimable_bytes);
1210 let pressure_remaining = pressure_paths
1211 .iter()
1212 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1213 if !report.failures.is_empty()
1214 || disk.worktree_limit_bytes > 0
1215 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1216 {
1217 let details = report
1218 .retained
1219 .iter()
1220 .filter(|reason| {
1221 !reason.contains("within policy")
1222 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1223 })
1224 .take(4)
1225 .chain(report.failures.iter().take(2))
1226 .cloned()
1227 .collect::<Vec<_>>()
1228 .join("; ");
1229 crate::notices::raise_in(
1230 home,
1231 crate::notices::Notice::warn(
1232 "disk:retention",
1233 format!(
1234 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1235 report.measured_bytes,
1236 report.proven_reclaimable_bytes,
1237 report.protected_or_unproven_bytes,
1238 report.freed_bytes,
1239 if report.proven_reclaimable_bytes == 0 {
1240 "No inspected contents have deletion authority."
1241 } else {
1242 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1243 }
1244 ),
1245 ),
1246 );
1247 }
1248 report.observed_home_free_change_bytes = free_before
1249 .zip(crate::disk::free_bytes(home).ok())
1250 .map(|(before, after)| {
1251 (i128::from(after) - i128::from(before))
1252 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1253 });
1254 let store = home.join("retention.json");
1255 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1256 let tmp = home.join("retention.json.tmp");
1257 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1258 tracing::warn!("retention report could not be saved: {e}");
1259 }
1260 }
1261 tracing::info!(
1262 measured_bytes = report.measured_bytes,
1263 freed_bytes = report.freed_bytes,
1264 retired = report.retired,
1265 retained = report.retained.len(),
1266 failures = report.failures.len(),
1267 "retained worktree cleanup"
1268 );
1269 for reason in report.retained.iter().chain(&report.failures) {
1270 tracing::debug!(%reason, "worktree retained");
1271 }
1272 report
1273}
1274
1275async fn retire_empty(
1276 home: &Path,
1277 disk: &crate::config::Disk,
1278 now: Timestamp,
1279 report: &mut Report,
1280) {
1281 if !disk.auto_fold {
1282 return;
1283 }
1284 let queue = crate::queue::Queue::at(home.join("queue"));
1285 let questions = crate::ask::Questions::at(home.join("questions"));
1286 let open = questions.list();
1287 for listed in queue.list() {
1288 let Ok(_claim) = queue.claim(&listed.id) else {
1289 continue;
1290 };
1291 let Ok(task) = queue.get(&listed.id) else {
1292 continue;
1293 };
1294 if open
1295 .iter()
1296 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1297 {
1298 report.retained.push(format!(
1299 "{}: task has an unanswered owner decision",
1300 task.id
1301 ));
1302 continue;
1303 }
1304 let sources: Vec<_> = task
1305 .runs
1306 .iter()
1307 .collect::<std::collections::BTreeSet<_>>()
1308 .into_iter()
1309 .cloned()
1310 .collect();
1311 let mut useful = Vec::new();
1312 let mut selected = Vec::new();
1313 let mut public = Vec::new();
1314 let mut unreadable = false;
1315 for id in &sources {
1316 let Ok(state) = RunState::load_under(id, home) else {
1317 unreadable = true;
1318 continue;
1319 };
1320 if state.released() || state.retired_to.is_some() {
1321 continue;
1322 }
1323 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1324 selected.push(state.clone());
1325 }
1326 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1327 public.push(state.clone());
1328 }
1329 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1330 useful.push(state);
1331 }
1332 }
1333 let pool = if !selected.is_empty() {
1337 selected
1338 } else if !public.is_empty() {
1339 public
1340 } else {
1341 useful
1342 };
1343 if unreadable || pool.len() != 1 {
1344 report.retained.push(format!(
1345 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1346 task.id,
1347 sources.len()
1348 ));
1349 continue;
1350 }
1351 let authority = &pool[0];
1352 if authority.candidates.is_empty() {
1353 continue;
1354 }
1355 let latest = &authority.id;
1356 for id in &sources {
1357 if id == latest || report.retired >= 5 {
1358 continue;
1359 }
1360 if let Ok(state) = RunState::load_under(id, home)
1361 && state.retired_to.as_ref() == Some(latest)
1362 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1363 {
1364 continue;
1365 }
1366 let result = async {
1367 ensure!(
1368 !task
1369 .resume_override
1370 .as_ref()
1371 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1372 "operator-pinned recovery run"
1373 );
1374 ensure!(
1375 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1376 "selected replacement recovery source"
1377 );
1378 let _owner = crate::pr_head::owner_lock(home, id)?;
1379 let mut state = RunState::load_under(id, home)?;
1380 ensure!(
1381 state.repo == authority.repo,
1382 "repository differs from recovery authority"
1383 );
1384 ensure!(
1385 !crate::pr_recovery::explicit_sources_retained(home, id),
1386 "explicitly retained recovery source"
1387 );
1388 ensure!(
1389 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1390 == Liveness::Dead,
1391 "run live or ownership unknown"
1392 );
1393 let _driver = crate::driver::maintenance(home, id)?;
1394 ensure!(
1395 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1396 "within grace period"
1397 );
1398 ensure!(
1399 !open.iter().any(|q| q.run == *id && q.status.open()),
1400 "unanswered owner decision"
1401 );
1402 ensure!(
1403 state.pr.is_none()
1404 && state.pr_head.is_none()
1405 && state.pr_recovery.is_none()
1406 && state.land_approval.is_none()
1407 && state.merge_decision.is_none()
1408 && state.reviews.is_empty()
1409 && state.recovery.is_none()
1410 && state.candidate_preservation.is_empty()
1411 && state.interrupted_implementations.is_empty()
1412 && !state.released(),
1413 "PR ownership or recovery checkpoint"
1414 );
1415 let proof = prove_empty(home, &state, latest).await?;
1416 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1417 for path in &proof.paths {
1420 ensure!(
1421 git::worktree_remove_clean(&state.repo, path).await?,
1422 "Git refused clean worktree removal"
1423 );
1424 }
1425 Ok::<_, anyhow::Error>(())
1426 }
1427 .await;
1428 match result {
1429 Ok(()) => report.retired += 1,
1430 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1431 }
1432 }
1433 }
1434}
1435
1436struct EmptyProof {
1437 paths: Vec<PathBuf>,
1438 _outputs: Vec<OutputLease>,
1439 _cargo: Vec<File>,
1440}
1441
1442pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1445 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1446 let mut files = Vec::new();
1447 let mut blobs = Vec::new();
1448 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1449 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1450 let parts: Vec<_> = header.split_whitespace().collect();
1451 ensure!(
1452 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1453 "unsupported tracked source kind"
1454 );
1455 let path = worktree.join(name);
1456 let meta = std::fs::symlink_metadata(&path)?;
1457 ensure!(
1458 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1459 "noncanonical tracked source"
1460 );
1461 #[cfg(unix)]
1462 {
1463 use std::os::unix::fs::PermissionsExt as _;
1464 ensure!(
1465 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1466 "tracked executable mode differs from original base"
1467 );
1468 }
1469 path.to_str().context("non UTF-8 source")?;
1470 files.push(git::path_arg(&path));
1471 blobs.push(parts[2].to_owned());
1472 }
1473 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1474 let mut args = vec!["hash-object", "--no-filters", "--"];
1475 args.extend(paths.iter().map(String::as_str));
1476 let actual = git::git(worktree, &args).await?;
1477 ensure!(
1478 actual.lines().eq(expected.iter().map(String::as_str)),
1479 "tracked bytes differ from original base"
1480 );
1481 }
1482 Ok(())
1483}
1484
1485async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1486 let id = &state.id;
1487 let paths = worktrees(state).await?;
1488 let mut guards = Vec::new();
1489 let mut cargo_guards = Vec::new();
1490 for path in &paths {
1491 guards.push(OutputLease::take(path, true)?);
1492 ensure!(
1493 git::rev_parse(path, "HEAD").await? == state.base_commit,
1494 "unique or unrecorded commits"
1495 );
1496 ensure!(
1497 git::git(
1498 path,
1499 &[
1500 "--no-optional-locks",
1501 "status",
1502 "--porcelain",
1503 "--untracked-files=all"
1504 ]
1505 )
1506 .await?
1507 .is_empty(),
1508 "tracked or untracked source changes"
1509 );
1510 ensure!(
1511 git::git(
1512 path,
1513 &[
1514 "--no-optional-locks",
1515 "status",
1516 "--porcelain",
1517 "--untracked-files=all",
1518 "--ignored",
1519 "--",
1520 ".",
1521 ":(exclude)target"
1522 ]
1523 )
1524 .await?
1525 .is_empty(),
1526 "source changes, ignored files or build outputs"
1527 );
1528 tracked_base(path, &state.base_commit).await?;
1529 match std::fs::symlink_metadata(path.join("target")) {
1530 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1531 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1532 Err(error) => return Err(error.into()),
1533 }
1534 for checkpoint in [
1535 "rebase-merge",
1536 "rebase-apply",
1537 "sequencer",
1538 "BISECT_LOG",
1539 "REVERT_HEAD",
1540 "AUTO_MERGE",
1541 "MERGE_MSG",
1542 "CHERRY_PICK_HEAD",
1543 "MERGE_HEAD",
1544 "REBASE_HEAD",
1545 ] {
1546 let location = git::git(
1547 path,
1548 &[
1549 "rev-parse",
1550 "--path-format=absolute",
1551 "--git-path",
1552 checkpoint,
1553 ],
1554 )
1555 .await?;
1556 ensure!(
1557 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1558 "retained Git recovery metadata: {checkpoint}"
1559 );
1560 }
1561 let origin_path = git::git(
1564 path,
1565 &[
1566 "rev-parse",
1567 "--path-format=absolute",
1568 "--git-path",
1569 "ORIG_HEAD",
1570 ],
1571 )
1572 .await?;
1573 match std::fs::symlink_metadata(origin_path.trim()) {
1574 Ok(meta) => {
1575 ensure!(
1576 meta.is_file() && !meta.file_type().is_symlink(),
1577 "unknown ORIG_HEAD artifact"
1578 );
1579 git::rev_parse(path, "ORIG_HEAD")
1580 .await
1581 .context("unreadable ORIG_HEAD")?;
1582 }
1583 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1584 Err(e) => return Err(e.into()),
1585 }
1586 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1587 ensure!(
1588 git::git_raw(
1589 path,
1590 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1591 )
1592 .await?
1593 .ok(),
1594 "ORIG_HEAD retains commits outside the original base"
1595 );
1596 }
1597 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1598 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1599 ensure!(
1600 git::git_raw(
1601 path,
1602 &["merge-base", "--is-ancestor", head, &state.base_commit]
1603 )
1604 .await?
1605 .ok(),
1606 "worktree reflog retains commits outside the original base"
1607 );
1608 }
1609 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1610 ensure!(
1611 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1612 .await?
1613 .ok(),
1614 "unfinished Git operation"
1615 );
1616 }
1617 }
1618 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1621 return Ok(EmptyProof {
1622 paths,
1623 _outputs: guards,
1624 _cargo: cargo_guards,
1625 });
1626 }
1627 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1628 for candidate in &state.candidates {
1629 ensure!(
1630 (paths.contains(&candidate.worktree)
1631 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1632 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1633 "candidate recovery source"
1634 );
1635 }
1636 let pattern = format!("refs/heads/magi/{}/", state.short());
1637 let branches = git::git(
1638 &state.repo,
1639 &["for-each-ref", "--format=%(objectname)", &pattern],
1640 )
1641 .await?;
1642 ensure!(
1643 branches.lines().all(|head| head == state.base_commit),
1644 "unrecorded candidate branch changes"
1645 );
1646 Ok(EmptyProof {
1647 paths,
1648 _outputs: guards,
1649 _cargo: cargo_guards,
1650 })
1651}
1652
1653fn commit_retirement(
1658 home: &Path,
1659 queue: &crate::queue::Queue,
1660 expected: &crate::queue::Task,
1661 state: &mut RunState,
1662 authority: &str,
1663 sources: &[String],
1664 now: Timestamp,
1665) -> Result<()> {
1666 let _write = queue.lock_task(&expected.id)?;
1667 let current = queue.get(&expected.id)?;
1668 ensure!(
1669 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1670 "task changed during retirement proof; retaining worktrees for the current owner decision"
1671 );
1672 ensure!(
1673 !current
1674 .resume_override
1675 .as_ref()
1676 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1677 && !current
1678 .replacement
1679 .as_ref()
1680 .is_some_and(|pin| pin.run == state.id),
1681 "current owner selected this recovery source"
1682 );
1683 ensure!(
1684 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1685 "explicitly retained recovery source"
1686 );
1687 ensure!(
1688 !crate::ask::Questions::at(home.join("questions"))
1689 .list()
1690 .iter()
1691 .any(|q| q.status.open()
1692 && (q.run == state.id
1693 || q.run == current.id
1694 || current.blocked_by.contains(&q.id))),
1695 "owner decision arrived during retirement proof"
1696 );
1697 let dir = home.join("recovery-lineages");
1698 std::fs::create_dir_all(&dir)?;
1699 let tmp = dir.join(format!("{}.tmp", current.id));
1700 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1701 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1702 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1703 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1704 state.retired_to = Some(authority.to_owned());
1705 state.event(
1706 "retention",
1707 format!(
1708 "empty attempt retired to {authority}; recovery sources: {}",
1709 sources.join(", ")
1710 ),
1711 );
1712 state.save_under(home)?;
1713 Ok(())
1715}
1716
1717pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1721 if min_free == 0 {
1722 return std::future::pending().await;
1723 }
1724 loop {
1725 for path in &paths {
1726 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1727 match crate::disk::free_bytes(existing) {
1728 Ok(free) if free >= min_free / 2 => {}
1729 Ok(free) => {
1730 return format!(
1731 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1732 );
1733 }
1734 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1735 }
1736 }
1737 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1738 }
1739}
1740
1741#[cfg(all(test, unix))]
1742mod tests {
1743 use super::*;
1744 use crate::{
1745 config::{Config, Disk},
1746 queue::{Source, Task},
1747 run::{Candidate, RunStatus},
1748 };
1749
1750 struct Fx {
1751 _temp: tempfile::TempDir,
1752 home: PathBuf,
1753 repo: PathBuf,
1754 bay: PathBuf,
1755 base: String,
1756 }
1757 impl Fx {
1758 async fn new() -> Self {
1759 let temp = tempfile::tempdir().unwrap();
1760 let root = temp.path().canonicalize().unwrap();
1761 let home = root.join("home");
1762 let repo = root.join("repo");
1763 let bay = root.join("worktrees");
1764 std::fs::create_dir_all(&repo).unwrap();
1765 std::fs::create_dir_all(home.join("runs")).unwrap();
1766 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1767 git::git(&repo, &["config", "user.name", "Fixture"])
1768 .await
1769 .unwrap();
1770 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1771 .await
1772 .unwrap();
1773 git::git(&repo, &["config", "commit.gpgsign", "false"])
1774 .await
1775 .unwrap();
1776 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1777 std::fs::write(repo.join("source"), "base\n").unwrap();
1778 git::git(&repo, &["add", "."]).await.unwrap();
1779 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1780 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1781 Self {
1782 _temp: temp,
1783 home,
1784 repo,
1785 bay,
1786 base,
1787 }
1788 }
1789 async fn run(&self, tag: &str, useful: bool) -> RunState {
1790 let mut cfg = Config::default();
1791 cfg.graph.worktree_root = Some(self.bay.clone());
1792 let mut state = RunState::new(
1793 self.repo.clone(),
1794 "main".into(),
1795 self.base.clone(),
1796 "task".into(),
1797 cfg,
1798 );
1799 state.id = format!("20260101-000000-{tag}");
1800 state.status = RunStatus::Blocked;
1801 state.driver_exited = true;
1802 let wt = state.worktree_root().join("cand-A");
1803 let branch = format!("magi/{tag}/A");
1804 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1805 .await
1806 .unwrap();
1807 if useful {
1808 std::fs::write(wt.join("source"), tag).unwrap();
1809 git::git(&wt, &["add", "."]).await.unwrap();
1810 git::git(&wt, &["commit", "-m", "candidate change"])
1811 .await
1812 .unwrap();
1813 }
1814 state.candidates.push(Candidate {
1815 index: 0,
1816 label: 'A',
1817 agent: "fixture".into(),
1818 branch,
1819 worktree: wt,
1820 summary: String::new(),
1821 stat: String::new(),
1822 files: usize::from(useful),
1823 commits: usize::from(useful),
1824 empty: !useful,
1825 failed: None,
1826 verified_noop: None,
1827 duration_ms: 0,
1828 folded: false,
1829 });
1830 state.save_under(&self.home).unwrap();
1831 state
1832 }
1833 fn disk() -> Disk {
1834 Disk {
1835 worktree_limit_bytes: 1,
1836 worktree_reclaim_bytes: 1024 * 1024,
1837 fold_grace_secs: 0,
1838 min_free_bytes: 0,
1839 ..Disk::default()
1840 }
1841 }
1842 fn target(wt: &Path) {
1843 let target = wt.join("target");
1844 std::fs::create_dir_all(target.join("debug")).unwrap();
1845 let mut artifact = [1; 200];
1846 artifact[..4].copy_from_slice(b"\x7fELF");
1847 std::fs::write(target.join("debug/output"), artifact).unwrap();
1848 let output = target
1849 .join("debug/output")
1850 .display()
1851 .to_string()
1852 .replace('\\', "\\\\")
1853 .replace(' ', "\\ ");
1854 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1855 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1856 std::fs::write(
1857 target.join("CACHEDIR.TAG"),
1858 "Signature: 8a477f597d28d172789f06886806bc55\n",
1859 )
1860 .unwrap();
1861 std::fs::write(
1862 target.join(".rustc_info.json"),
1863 "{\"rustc_fingerprint\":123}",
1864 )
1865 .unwrap();
1866 }
1867 fn now() -> Timestamp {
1868 Timestamp::now() + jiff::SignedDuration::from_secs(2)
1869 }
1870 fn task(&self, runs: &[RunState]) -> Task {
1871 let mut task = Task::new(
1872 "fixture".into(),
1873 "task".into(),
1874 self.repo.clone(),
1875 Source::Human,
1876 );
1877 task.runs = runs.iter().map(|s| s.id.clone()).collect();
1878 crate::queue::Queue::at(self.home.join("queue"))
1879 .put(&mut task)
1880 .unwrap();
1881 task
1882 }
1883 }
1884
1885 #[cfg(unix)]
1886 #[tokio::test]
1887 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1888 let f = Fx::new().await;
1889 let state = f.run("ca01", true).await;
1890 let wt = &state.candidates[0].worktree;
1891 let generation = crate::incremental::tests::generate(wt);
1892 for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1894 for entry in std::fs::read_dir(dir).unwrap() {
1895 let path = entry.unwrap().path();
1896 if path.extension().is_some_and(|x| x == "d") {
1897 std::fs::remove_file(path).unwrap();
1898 }
1899 }
1900 }
1901 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1902 assert_eq!(
1903 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1904 0,
1905 "live generation is protected"
1906 );
1907 drop(generation);
1908 let candidates = crate::incremental::candidates(wt).unwrap();
1909 let logical: u64 = candidates
1910 .files
1911 .iter()
1912 .map(|p| std::fs::metadata(p).unwrap().len())
1913 .sum();
1914 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1915 std::fs::write(&unknown, "retain unknown data").unwrap();
1916 drop(candidates);
1917 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1918 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1919 assert!(
1920 refused
1921 .retained
1922 .iter()
1923 .any(|s| s.contains("provenance mismatch"))
1924 );
1925 std::fs::remove_file(&unknown).unwrap();
1926 let legacy = wt.join("target/debug/incremental/legacy-session");
1927 std::fs::create_dir(&legacy).unwrap();
1928 let candidates = crate::incremental::candidates(wt).unwrap();
1930 let bin = candidates
1931 .files
1932 .iter()
1933 .find(|p| p.extension().is_some_and(|x| x == "bin"))
1934 .unwrap();
1935 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1936 drop(candidates);
1937 let cargo = cargo_locks(&wt.join("target")).unwrap();
1938 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1939 drop(cargo);
1940 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1941 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1942 drop(outputs);
1943 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1944 assert_eq!(report.freed_bytes, logical, "{report:?}");
1945 assert_eq!(report.proven_reclaimable_bytes, logical);
1946 assert!(report.measured_bytes > logical);
1947 assert_eq!(
1948 report.protected_or_unproven_bytes,
1949 report.measured_bytes - logical
1950 );
1951 #[cfg(unix)]
1952 {
1953 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
1954 assert!(
1955 report.measured_allocated_file_bytes.unwrap()
1956 >= report.estimated_released_allocated_bytes.unwrap()
1957 );
1958 }
1959 assert!(report.observed_home_free_change_bytes.is_some());
1960 assert!(legacy.join("query-cache.bin").is_file());
1961 assert!(wt.join("src/lib.rs").is_file());
1962 assert_eq!(report.retired, 0);
1963 }
1964
1965 #[cfg(unix)]
1966 #[tokio::test]
1967 async fn incremental_and_deps_links_account_for_retained_third_link() {
1968 use std::os::unix::fs::MetadataExt;
1969 let f = Fx::new().await;
1970 let state = f.run("ca02", true).await;
1971 let wt = &state.candidates[0].worktree;
1972 let generation = crate::incremental::tests::generate(wt);
1973 drop(generation);
1974 let candidates = crate::incremental::candidates(wt).unwrap();
1975 let objects = candidates
1976 .files
1977 .iter()
1978 .filter(|p| p.extension().is_some_and(|x| x == "o"))
1979 .take(2)
1980 .cloned()
1981 .collect::<Vec<_>>();
1982 assert_eq!(
1983 objects.len(),
1984 2,
1985 "fixture must produce real incremental objects"
1986 );
1987 let mut outputs = candidates.files.clone();
1988 drop(candidates);
1989 let target = wt.join("target");
1990 for dir in [target.join("debug"), target.join("debug/deps")] {
1991 for entry in std::fs::read_dir(dir).unwrap() {
1992 let path = entry.unwrap().path();
1993 if path.extension().is_some_and(|x| x == "d") {
1994 std::fs::remove_file(path).unwrap();
1995 }
1996 }
1997 }
1998 for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
2001 let path = entry.unwrap().path();
2002 if path.is_file() {
2003 std::fs::remove_file(path).unwrap();
2004 }
2005 }
2006 for (index, object) in objects.iter().enumerate() {
2007 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2008 let dep = target.join(format!("debug/deps/link{index}"));
2009 std::fs::hard_link(object, &dep).unwrap();
2010 std::fs::write(
2011 dep.with_extension("d"),
2012 format!("{}: src/lib.rs\n", dep.display()),
2013 )
2014 .unwrap();
2015 outputs.push(dep);
2016 }
2017 let retained = wt.join("retained-input");
2018 std::fs::hard_link(&objects[1], &retained).unwrap();
2019 let sentinel = std::fs::read(&retained).unwrap();
2020 let logical: u64 = outputs
2021 .iter()
2022 .map(|p| std::fs::metadata(p).unwrap().len())
2023 .sum();
2024 let mut seen = std::collections::BTreeSet::new();
2025 let retained_meta = std::fs::metadata(&retained).unwrap();
2026 let retained_inode = (retained_meta.dev(), retained_meta.ino());
2027 let mut expected = 0u64;
2028 for output in &outputs {
2029 let m = std::fs::metadata(output).unwrap();
2030 let inode = (m.dev(), m.ino());
2031 if inode != retained_inode && seen.insert(inode) {
2032 expected += m.blocks() * 512;
2033 }
2034 }
2035 assert!(expected > 0);
2036 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2037 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2038 assert_eq!(report.freed_bytes, logical, "{report:?}");
2039 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2040 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2041 assert!(outputs.iter().all(|p| !p.exists()));
2042 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2043 assert!(wt.join("src/lib.rs").is_file());
2044 }
2045
2046 #[tokio::test]
2047 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2048 let f = Fx::new().await;
2049 let state = f.run("ca03", true).await;
2050 let wt = &state.candidates[0].worktree;
2051 Fx::target(wt);
2052 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2053 let unknown = wt.join("target/debug/operator-note");
2054 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2055 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2056 assert_eq!(report.proven_reclaimable_bytes, 0);
2057 #[cfg(unix)]
2058 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2059 assert!(report.protected_or_unproven_bytes >= 4096);
2060 let message = || {
2061 crate::notices::Notices::at(f.home.join("notifications"))
2062 .list()
2063 .into_iter()
2064 .find(|n| n.key == "disk:retention")
2065 .unwrap()
2066 .message
2067 };
2068 let notice = message();
2069 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2070 assert!(notice.contains("protected or unproven"), "{notice}");
2071 assert!(
2072 notice.contains("legacy incremental/build and unknown files retained"),
2073 "{notice}"
2074 );
2075 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2076 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2077 assert_eq!(report.freed_bytes, 0);
2078 let notice = message();
2079 assert!(notice.contains(&state.id), "{notice}");
2080 assert!(
2081 notice.contains("lock") || notice.contains("owner"),
2082 "{notice}"
2083 );
2084 drop(owner);
2085 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2086 }
2087
2088 #[tokio::test]
2089 async fn real_cargo_build_holds_the_target_against_reclamation() {
2090 let f = Fx::new().await;
2091 let state = f.run("ca02", true).await;
2092 let wt = &state.candidates[0].worktree;
2093 Fx::target(wt);
2094 std::fs::create_dir(wt.join("src")).unwrap();
2095 std::fs::write(
2096 wt.join("Cargo.toml"),
2097 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2098 )
2099 .unwrap();
2100 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2101 std::fs::write(
2102 wt.join("build.rs"),
2103 r#"fn main() {
2104 let ready = std::env::var("FIXTURE_READY").unwrap();
2105 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2106 std::fs::write(ready, "ready").unwrap();
2107 let start = std::time::Instant::now();
2108 while !std::path::Path::new(&release).exists() && start.elapsed().as_secs() < 15 {
2109 std::thread::sleep(std::time::Duration::from_millis(10));
2110 }
2111 }"#,
2112 )
2113 .unwrap();
2114 let ready = wt.join("ready");
2115 let release = wt.join("release");
2116 let mut cargo = tokio::process::Command::new("cargo");
2117 cargo.env_clear();
2118 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2119 if let Some(value) = std::env::var_os(key) {
2120 cargo.env(key, value);
2121 }
2122 }
2123 let mut child = cargo
2124 .current_dir(wt)
2125 .env("RUSTUP_TOOLCHAIN", "stable")
2126 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2127 .env("CARGO_INCREMENTAL", "1")
2128 .env("FIXTURE_READY", &ready)
2129 .env("FIXTURE_RELEASE", &release)
2130 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2131 .arg(wt.join("target"))
2132 .stdout(std::process::Stdio::null())
2133 .stderr(std::process::Stdio::null())
2134 .kill_on_drop(true)
2135 .spawn()
2136 .unwrap();
2137 let arrived = tokio::time::timeout(std::time::Duration::from_secs(10), async {
2138 while !ready.exists() {
2139 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2140 }
2141 })
2142 .await
2143 .is_ok();
2144 let report = if arrived {
2145 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2146 } else {
2147 None
2148 };
2149 std::fs::write(release, "release").unwrap();
2151 let status = tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await;
2152 if status.is_err() {
2153 child.kill().await.unwrap();
2154 child.wait().await.unwrap();
2155 }
2156 assert!(
2157 arrived,
2158 "fixture build never reached its active build script"
2159 );
2160 assert!(status.unwrap().unwrap().success());
2161 let report = report.unwrap();
2162 assert_eq!(report.freed_bytes, 0, "{report:?}");
2163 assert!(
2164 report
2165 .retained
2166 .iter()
2167 .any(|s| s.contains("Cargo build is active")),
2168 "{report:?}"
2169 );
2170 assert!(wt.join("target/debug/output").is_file());
2171 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2172 }
2173
2174 #[cfg(unix)]
2175 #[test]
2176 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2177 let temp = tempfile::tempdir().unwrap();
2178 let a = temp.path().join("a");
2179 let b = temp.path().join("b");
2180 let outside = temp.path().join("retained-input");
2181 std::fs::write(&a, vec![1; 65536]).unwrap();
2182 std::fs::hard_link(&a, &b).unwrap();
2183 std::fs::hard_link(&a, &outside).unwrap();
2184 let mut seen = Default::default();
2185 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2186 assert!(blocks > 0);
2187 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2188 std::fs::remove_file(&a).unwrap();
2189 assert_eq!(accounting.unlinked(&a), Some(0));
2190 std::fs::remove_file(&b).unwrap();
2191 assert_eq!(accounting.unlinked(&b), Some(0));
2192 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2193 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2194 std::fs::remove_file(&outside).unwrap();
2195 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2196 }
2197
2198 #[cfg(unix)]
2199 #[test]
2200 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2201 let temp = tempfile::tempdir_in(".").unwrap();
2202 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2203 assert!(home.is_relative());
2204 let paths = vec![std::env::current_dir().unwrap()];
2205 let gib = 1024 * 1024 * 1024;
2206 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2207 assert!(
2208 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2209 9 * gib
2210 ))
2211 .is_err()
2212 );
2213 drop(reservation);
2214 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2215 let bad = temp.path().join("symlink-state");
2216 std::fs::create_dir(&bad).unwrap();
2217 std::os::unix::fs::symlink(
2218 home.canonicalize().unwrap().join("disk-admission"),
2219 bad.join("disk-admission"),
2220 )
2221 .unwrap();
2222 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2223 }
2224
2225 #[cfg(unix)]
2226 #[test]
2227 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2228 let temp = tempfile::tempdir().unwrap();
2229 let home = temp.path().canonicalize().unwrap();
2230 let paths = vec![home.clone(), home.join("not-created-yet")];
2231 let gib = 1024 * 1024 * 1024;
2232 let floor = 8 * gib;
2233 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2234 assert_eq!(first.len(), 1);
2235 let blocked =
2236 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2237 assert!(blocked.to_string().contains("active writer reservations"));
2238 let second =
2239 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2240 drop(first);
2241 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2242 drop(second);
2243 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2244 assert!(
2245 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2246 );
2247 }
2248
2249 #[cfg(unix)]
2250 #[test]
2251 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2252 let temp = tempfile::tempdir().unwrap();
2253 let home = temp.path().canonicalize().unwrap();
2254 let barrier = std::sync::Barrier::new(2);
2255 let gib = 1024 * 1024 * 1024;
2256 let results = std::thread::scope(|scope| {
2257 let threads: Vec<_> = (0..2)
2258 .map(|_| {
2259 scope.spawn(|| {
2260 barrier.wait();
2261 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2262 Ok(100 + gib)
2263 })
2264 })
2265 })
2266 .collect();
2267 threads
2268 .into_iter()
2269 .map(|thread| thread.join().unwrap())
2270 .collect::<Vec<_>>()
2271 });
2272 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2273 assert!(
2274 results
2275 .iter()
2276 .filter_map(|r| r.as_ref().err())
2277 .all(|e| e.to_string().contains("active writer reservations"))
2278 );
2279 }
2280
2281 #[tokio::test]
2282 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2283 real_cargo_fixture(false).await;
2284 }
2285
2286 #[tokio::test]
2287 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2288 real_cargo_fixture(true).await;
2289 }
2290
2291 async fn real_cargo_fixture(precreate: bool) {
2292 let f = Fx::new().await;
2293 let state = f.run("ca00", true).await;
2294 let wt = &state.candidates[0].worktree;
2295 let source = "fn main() { println!(\"retained fixture\"); }\n";
2296 std::fs::create_dir(wt.join("src")).unwrap();
2297 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2298 std::fs::write(
2299 wt.join("Cargo.toml"),
2300 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2301 )
2302 .unwrap();
2303 if precreate {
2304 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2305 }
2306 let mut cargo = std::process::Command::new("cargo");
2307 cargo.env_clear();
2308 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2309 if let Some(value) = std::env::var_os(key) {
2310 cargo.env(key, value);
2311 }
2312 }
2313 let build = cargo
2314 .current_dir(wt)
2315 .env("RUSTUP_TOOLCHAIN", "stable")
2316 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2317 .env("CARGO_INCREMENTAL", "0")
2318 .args([
2319 "build",
2320 "--offline",
2321 "--release",
2322 "--jobs",
2323 "1",
2324 "--target-dir",
2325 ])
2326 .arg(wt.join("target"))
2327 .output()
2328 .unwrap();
2329 assert!(
2330 build.status.success(),
2331 "{}",
2332 String::from_utf8_lossy(&build.stderr)
2333 );
2334 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2335 if precreate {
2336 let again = cargo.output().unwrap();
2338 assert!(
2339 again.status.success(),
2340 "{}",
2341 String::from_utf8_lossy(&again.stderr)
2342 );
2343 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2344 assert!(cargo_target(wt).await.is_err());
2345 }
2346 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2347 .await
2348 .unwrap();
2349 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2350 .await
2351 .unwrap();
2352 let artifact = wt.join("target/release/retained-fixture");
2353 assert!(artifact.is_file());
2354 let unknown = wt.join("target/release/operator-note");
2355 std::fs::write(&unknown, "keep this note\n").unwrap();
2356
2357 let target = wt.join("target");
2358 let mut preserved = std::collections::BTreeMap::new();
2359 for relative in [
2360 "tmp/sentinel",
2361 "magi-tools/sentinel",
2362 "lifecycle-client-check.log",
2363 ] {
2364 let path = target.join(relative);
2365 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2366 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2367 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2368 }
2369 let decoy = target.join("magi-tools/retained-fixture");
2372 std::fs::copy(&artifact, &decoy).unwrap();
2373 let dep_info = decoy.with_extension("d");
2374 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2375 for path in [
2376 decoy,
2377 dep_info,
2378 wt.join("Cargo.toml"),
2379 wt.join("Cargo.lock"),
2380 ] {
2381 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2382 }
2383 let proven = declared_outputs(&target).unwrap();
2384
2385 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2386 assert!(out.freed_bytes > 0, "{out:?}");
2387 for (path, bytes) in preserved {
2388 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2389 }
2390 assert!(!artifact.exists(), "{out:?}");
2391 assert_eq!(
2392 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2393 source
2394 );
2395 assert_eq!(
2396 std::fs::read_to_string(&unknown).unwrap(),
2397 "keep this note\n"
2398 );
2399 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2400 let ledger: serde_json::Value = serde_json::from_slice(
2401 &std::fs::read(
2402 f.home
2403 .join("runs")
2404 .join(&state.id)
2405 .join("build-outputs.json"),
2406 )
2407 .unwrap(),
2408 )
2409 .unwrap();
2410 assert_eq!(
2411 ledger["identity"],
2412 if precreate {
2413 "rustc-profile-dep-info"
2414 } else {
2415 "cache-tag"
2416 }
2417 );
2418 if precreate {
2419 assert!(!target.join("CACHEDIR.TAG").exists());
2420 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2421 }
2422 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2423 assert!(deleted.contains(&artifact));
2424 assert_eq!(deleted, proven);
2425 for path in deleted {
2426 assert!(!path.exists());
2427 assert!(
2428 [target.join("release"), target.join("release/deps")]
2429 .contains(&path.parent().unwrap().to_owned())
2430 );
2431 }
2432 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2433 }
2434
2435 fn missing_tag_target(wt: &Path) {
2436 Fx::target(wt);
2437 let target = wt.join("target");
2438 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2439 for subdir in ["debug/deps", "debug/.fingerprint"] {
2440 std::fs::create_dir_all(target.join(subdir)).unwrap();
2441 }
2442 std::fs::write(target.join(".rustc_info.json"),
2443 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2444 }
2445
2446 #[tokio::test]
2447 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2448 for legacy in [false, true] {
2449 let f = Fx::new().await;
2450 let mut state = f.run("ac00", true).await;
2451 Fx::target(&state.candidates[0].worktree);
2452 let original = state.updated_at;
2453 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2454 if legacy {
2455 let path = f.home.join("runs").join(&state.id).join("run.json");
2456 let mut value = serde_json::to_value(&state).unwrap();
2457 value.as_object_mut().unwrap().remove("output_activity_at");
2458 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2459 state = RunState::load_under(&state.id, &f.home).unwrap();
2460 assert_eq!(state.output_activity(), original);
2461 }
2462 let age = state.output_activity();
2463 for offset in [30, 20, 10] {
2464 state
2465 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2466 .unwrap();
2467 state = RunState::load_under(&state.id, &f.home).unwrap();
2468 assert_eq!(state.output_activity(), age);
2469 }
2470 let mut disk = Fx::disk();
2471 disk.fold_grace_secs = 3600;
2472 let report = sweep(&f.home, &disk, now).await;
2473 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2474 }
2475 }
2476
2477 #[tokio::test]
2478 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2479 for evidence in [
2480 "execution",
2481 "artifact",
2482 "dep-info",
2483 "fingerprint",
2484 "lease",
2485 "future",
2486 ] {
2487 let f = Fx::new().await;
2488 let mut state = f.run("ac01", true).await;
2489 let wt = state.candidates[0].worktree.clone();
2490 missing_tag_target(&wt);
2491 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2492 let recent = now - jiff::SignedDuration::from_secs(10);
2493 match evidence {
2494 "execution" => {
2495 state.note_output_activity(recent);
2496 state.save_under_at(&f.home, recent).unwrap();
2497 }
2498 "lease" => {
2499 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2500 let file = lease._file.try_clone().unwrap();
2501 drop(lease);
2502 file.set_modified(std::time::SystemTime::from(recent))
2503 .unwrap();
2504 }
2505 _ => {
2506 let relative = match evidence {
2507 "dep-info" => "debug/output.d",
2508 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2509 _ => "debug/output",
2510 };
2511 let path = wt.join("target").join(relative);
2512 let file = OpenOptions::new()
2513 .write(true)
2514 .create(true)
2515 .truncate(false)
2516 .open(path)
2517 .unwrap();
2518 let at = if evidence == "future" {
2519 now + jiff::SignedDuration::from_secs(10)
2520 } else {
2521 recent
2522 };
2523 file.set_modified(std::time::SystemTime::from(at)).unwrap();
2524 }
2525 }
2526 let mut disk = Fx::disk();
2527 disk.fold_grace_secs = 3600;
2528 let report = sweep(&f.home, &disk, now).await;
2529 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2530 assert!(
2531 report.retained.iter().any(|s| s.contains("grace period")
2532 || s.contains("execution activity after inventory")),
2533 "{evidence}: {report:?}"
2534 );
2535 let aged = now + jiff::SignedDuration::from_secs(7200);
2536 let report = sweep(&f.home, &disk, aged).await;
2537 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2538 }
2539 }
2540
2541 #[tokio::test]
2542 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2543 let f = Fx::new().await;
2544 let state = f.run("ac02", true).await;
2545 let wt = &state.candidates[0].worktree;
2546 missing_tag_target(wt);
2547 let target = wt.join("target");
2548 let unknown = target.join("debug/notes");
2549 std::fs::write(&unknown, "operator data").unwrap();
2550 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2551 let path = target.join("debug").join(lock);
2552 let file = OpenOptions::new()
2553 .create(true)
2554 .truncate(false)
2555 .read(true)
2556 .write(true)
2557 .open(&path)
2558 .unwrap();
2559 crate::pr_head::try_lock_file(&file).unwrap();
2560 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2561 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2562 assert!(
2563 report
2564 .retained
2565 .iter()
2566 .any(|s| s.contains(path.to_str().unwrap())),
2567 "{report:?}"
2568 );
2569 }
2570 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2571 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2572 drop(owner);
2573 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2574 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2575 drop(driver);
2576 let output = OutputLease::borrow(wt).unwrap().unwrap();
2577 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2578 drop(output);
2579 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2580 assert_eq!(report.freed_bytes, 200, "{report:?}");
2581 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2582 assert!(!target.join("CACHEDIR.TAG").exists());
2583 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2584 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2585 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2586 }
2587
2588 #[tokio::test]
2589 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2590 let f = Fx::new().await;
2591 let state = f.run("ac04", true).await;
2592 let wt = &state.candidates[0].worktree;
2593 missing_tag_target(wt);
2594 let target = wt.join("target");
2595 let first = target.join("debug/output");
2596 let second = target.join("debug/second");
2597 std::fs::copy(&first, &second).unwrap();
2598 std::fs::write(
2599 second.with_extension("d"),
2600 format!("{}: source\n", second.display()),
2601 )
2602 .unwrap();
2603 let outputs = declared_outputs(&target).unwrap();
2604 assert_eq!(outputs.len(), 2);
2605 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2607 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2608 std::fs::remove_file(first).unwrap();
2609 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2610 assert_eq!(report.freed_bytes, 200, "{report:?}");
2611 assert!(!second.exists());
2612 assert!(second.with_extension("d").exists());
2613 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2614 }
2615
2616 #[tokio::test]
2617 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2618 for invalid in [
2619 "identity",
2620 "deps",
2621 "fingerprint",
2622 "lock",
2623 "dep-info",
2624 "tag-symlink",
2625 "bad-tag",
2626 ] {
2627 let f = Fx::new().await;
2628 let state = f.run("ac03", true).await;
2629 let wt = &state.candidates[0].worktree;
2630 missing_tag_target(wt);
2631 let target = wt.join("target");
2632 match invalid {
2633 "identity" => std::fs::write(
2634 target.join(".rustc_info.json"),
2635 r#"{"rustc_fingerprint":123}"#,
2636 )
2637 .unwrap(),
2638 "deps" | "fingerprint" => {
2639 let path = target.join("debug").join(if invalid == "deps" {
2640 "deps"
2641 } else {
2642 ".fingerprint"
2643 });
2644 std::fs::remove_dir(&path).unwrap();
2645 std::os::unix::fs::symlink(&f.repo, path).unwrap();
2646 }
2647 "lock" => {
2648 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2649 }
2650 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2651 "tag-symlink" => {
2652 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2653 .unwrap()
2654 }
2655 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2656 _ => unreachable!(),
2657 }
2658 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2659 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2660 assert!(target.join("debug/output").is_file());
2661 }
2662 }
2663
2664 #[tokio::test]
2665 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2666 let f = Fx::new().await;
2667 let first = f.run("ba00", true).await;
2668 let second = f.run("ba01", true).await;
2669 for state in [&first, &second] {
2670 Fx::target(&state.candidates[0].worktree);
2671 }
2672 let mut disk = Fx::disk();
2673 disk.worktree_reclaim_bytes = 100;
2674 let out = sweep(&f.home, &disk, Fx::now()).await;
2675 assert_eq!(out.freed_bytes, 200, "{out:?}");
2676 assert!(
2677 out.retained
2678 .contains(&format!("{}: per-pass byte budget spent", second.id)),
2679 "{out:?}"
2680 );
2681 assert!(
2682 !first.candidates[0]
2683 .worktree
2684 .join("target/debug/output")
2685 .exists()
2686 );
2687 assert!(
2688 second.candidates[0]
2689 .worktree
2690 .join("target/debug/output")
2691 .exists()
2692 );
2693 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2694 }
2695
2696 #[tokio::test]
2697 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2698 let f = Fx::new().await;
2699 let small = f.run("bb00", true).await;
2700 let large = f.run("bb01", true).await;
2701 for state in [&small, &large] {
2702 Fx::target(&state.candidates[0].worktree);
2703 }
2704 let output = large.candidates[0].worktree.join("target/debug/output");
2705 std::fs::OpenOptions::new()
2706 .write(true)
2707 .open(&output)
2708 .unwrap()
2709 .set_len(1000)
2710 .unwrap();
2711 let mut disk = Fx::disk();
2712 disk.worktree_reclaim_bytes = 300;
2713 disk.min_free_bytes = u64::MAX;
2714 disk.worktree_limit_bytes = u64::MAX;
2716 let out = sweep(&f.home, &disk, Fx::now()).await;
2717 assert_eq!(out.freed_bytes, 1000, "{out:?}");
2718 assert!(!output.exists());
2719 assert!(
2720 small.candidates[0]
2721 .worktree
2722 .join("target/debug/output")
2723 .exists()
2724 );
2725 }
2726
2727 #[tokio::test]
2728 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2729 let f = Fx::new().await;
2730 let first = f.run("bc00", true).await;
2731 let second = f.run("bc01", true).await;
2732 for state in [&first, &second] {
2733 Fx::target(&state.candidates[0].worktree);
2734 }
2735 std::fs::create_dir_all(
2737 first.candidates[0]
2738 .worktree
2739 .join("target/debug/incremental"),
2740 )
2741 .unwrap();
2742 std::fs::write(
2743 first.candidates[0]
2744 .worktree
2745 .join("target/debug/incremental/cache"),
2746 [0; 1000],
2747 )
2748 .unwrap();
2749 let mut disk = Fx::disk();
2750 disk.worktree_reclaim_bytes = 0;
2751 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2752 disk.worktree_reclaim_bytes = 300;
2753 let out = sweep(&f.home, &disk, Fx::now()).await;
2754 assert_eq!(out.freed_bytes, 200, "{out:?}");
2755 assert!(
2756 !first.candidates[0]
2757 .worktree
2758 .join("target/debug/output")
2759 .exists()
2760 );
2761 assert!(
2762 second.candidates[0]
2763 .worktree
2764 .join("target/debug/output")
2765 .exists()
2766 );
2767 assert!(
2768 out.retained
2769 .iter()
2770 .any(|r| r.contains("declared outputs exceed"))
2771 );
2772 }
2773
2774 #[tokio::test]
2775 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2776 let f = Fx::new().await;
2777 let mut protected = f.run("bd00", true).await;
2778 let later = f.run("bd01", true).await;
2779 for state in [&protected, &later] {
2780 Fx::target(&state.candidates[0].worktree);
2781 }
2782 let mut disk = Fx::disk();
2783 disk.worktree_reclaim_bytes = 100;
2784 disk.fold_grace_secs = 3600;
2785 let out = sweep(&f.home, &disk, Fx::now()).await;
2786 assert_eq!(out.freed_bytes, 0);
2787 assert!(
2788 out.retained
2789 .iter()
2790 .any(|r| r.contains("within grace period"))
2791 );
2792 disk.fold_grace_secs = 0;
2793 protected.driver_exited = false;
2794 protected.driver_pid = Some(std::process::id());
2795 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2796 protected
2797 .save_under_at(&f.home, protected.updated_at)
2798 .unwrap();
2799 let out = sweep(&f.home, &disk, Fx::now()).await;
2800 assert!(
2801 out.retained
2802 .iter()
2803 .any(|r| r.contains("run live or ownership unknown"))
2804 );
2805 assert_eq!(
2806 out.freed_bytes, 200,
2807 "live target must not consume exception: {out:?}"
2808 );
2809 let output = protected.candidates[0].worktree.join("target/debug/output");
2810 assert!(output.exists());
2811 protected.driver_exited = true;
2812 protected.save_under(&f.home).unwrap();
2813 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2814 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2815 assert!(output.exists());
2816 drop(owner);
2817 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2818 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2819 assert!(output.exists());
2820 drop(cargo);
2821 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2822 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2823 assert!(output.exists());
2824 drop(lease);
2825 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2826 assert!(!output.exists());
2827 }
2828
2829 #[tokio::test]
2830 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2831 for remove_root in [true, false] {
2832 let f = Fx::new().await;
2833 let stale = Fx::new().await;
2834 let mut missing = stale.run("aa00", false).await;
2835 Fx::target(&missing.candidates[0].worktree);
2836 missing.save_under(&f.home).unwrap();
2837 if remove_root {
2838 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2839 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2842 } else {
2843 std::fs::remove_dir_all(&stale.repo).unwrap();
2844 }
2845 let healthy = f.run("aa01", false).await;
2846 Fx::target(&healthy.candidates[0].worktree);
2847
2848 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2849 assert!(report.failures.is_empty(), "{report:?}");
2850 assert_eq!(report.freed_bytes, 200, "{report:?}");
2851 assert!(
2852 !healthy.candidates[0]
2853 .worktree
2854 .join("target/debug/output")
2855 .exists()
2856 );
2857 if !remove_root {
2858 assert!(
2860 missing.candidates[0]
2861 .worktree
2862 .join("target/debug/output")
2863 .exists()
2864 );
2865 }
2866 }
2867 }
2868
2869 #[tokio::test]
2870 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2871 let f = Fx::new().await;
2872 let mut missing = f.run("aa00", false).await;
2873 Fx::target(&missing.candidates[0].worktree);
2874 missing.repo = missing.candidates[0].worktree.clone();
2875 missing.save_under(&f.home).unwrap();
2876 std::fs::remove_dir_all(&missing.repo).unwrap();
2877 assert!(missing.worktree_root().is_dir());
2878 assert!(f.repo.is_dir());
2879
2880 let healthy = f.run("aa01", false).await;
2881 Fx::target(&healthy.candidates[0].worktree);
2882
2883 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2884 assert!(report.failures.is_empty(), "{report:?}");
2885 assert_eq!(report.freed_bytes, 200, "{report:?}");
2886 assert!(
2887 !healthy.candidates[0]
2888 .worktree
2889 .join("target/debug/output")
2890 .exists()
2891 );
2892 }
2893
2894 #[tokio::test]
2895 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2896 let f = Fx::new().await;
2897 let broken = Fx::new().await;
2898 let mut invalid = broken.run("aa00", false).await;
2899 Fx::target(&invalid.candidates[0].worktree);
2900 invalid.save_under(&f.home).unwrap();
2901 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2902 let healthy = f.run("aa01", false).await;
2903 Fx::target(&healthy.candidates[0].worktree);
2904
2905 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2906 assert_eq!(report.failures.len(), 1, "{report:?}");
2907 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2908 assert_eq!(report.freed_bytes, 200, "{report:?}");
2909 assert!(
2910 !healthy.candidates[0]
2911 .worktree
2912 .join("target/debug/output")
2913 .exists()
2914 );
2915 assert!(
2916 invalid.candidates[0]
2917 .worktree
2918 .join("target/debug/output")
2919 .exists()
2920 );
2921 }
2922
2923 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2924 let path = reconcile::run(&f.home, &task.id, &source.id, None)
2925 .await
2926 .unwrap();
2927 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2928 }
2929
2930 #[tokio::test]
2931 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
2932 use crate::{
2933 chat_control::{Operation, dispatch},
2934 talk::{self, Talks},
2935 };
2936 let f = Fx::new().await;
2937 let source = f.run("ae00", true).await;
2938 let clean = f.run("ae01", false).await;
2939 let queue = crate::queue::Queue::at(f.home.join("queue"));
2940 let talks = Talks::at(f.home.join("talks"));
2941 let open_talk = |id: &str| -> talk::Talk {
2942 let mut conversation = serde_json::from_value(serde_json::json!({
2943 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
2944 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
2945 "seat":crate::agent::SeatState::new("talk","worker",1)
2946 }))
2947 .unwrap();
2948 talks.put(&mut conversation).unwrap();
2949 conversation
2950 };
2951 let conversation = open_talk("talk-owner");
2952 let other = open_talk("talk-other");
2953 let task = f.task(&[source.clone(), clean.clone()]);
2955 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2956 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
2957 task: task.id.clone(),
2958 authority: source.id.clone(),
2959 apply: apply.map(str::to_owned),
2960 };
2961 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
2963 .await
2964 .unwrap();
2965 let legacy: serde_json::Value =
2966 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
2967 let legacy_id = legacy["id"].as_str().unwrap();
2968 assert!(
2969 dispatch(
2970 &queue,
2971 &f.home,
2972 &talks,
2973 &conversation.id,
2974 operation(Some(legacy_id))
2975 )
2976 .await
2977 .is_err()
2978 );
2979 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
2980 .await
2981 .unwrap();
2982 assert!(summary.contains("Complete review summary"));
2983 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
2984 let root = f.home.join("retention-reconciliations");
2985 let displayed: serde_json::Value =
2986 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
2987 let id = displayed["plan"].as_str().unwrap();
2988 let dir = root.join(id);
2989 let plan: serde_json::Value =
2990 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
2991 assert_eq!(plan["version"], 2);
2992 assert!(clean.candidates[0].worktree.exists());
2993 assert!(!dir.join("approved.json").exists());
2994 assert!(
2996 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
2997 .await
2998 .is_err()
2999 );
3000 assert!(!dir.join("approved.json").exists());
3001 let error = reconcile::interrupt_chat_after_fence(
3004 &f.home,
3005 &task,
3006 id,
3007 reconcile::ChatRequest {
3008 talks: &talks,
3009 conversation: &conversation.id,
3010 },
3011 )
3012 .await
3013 .unwrap_err();
3014 assert!(
3015 error.to_string().contains("simulated interruption"),
3016 "{error:#}"
3017 );
3018 assert!(clean.candidates[0].worktree.exists());
3019 assert!(dir.join("approved.json").exists());
3020 dispatch(
3021 &queue,
3022 &f.home,
3023 &talks,
3024 &conversation.id,
3025 operation(Some(id)),
3026 )
3027 .await
3028 .unwrap();
3029 assert!(!clean.candidates[0].worktree.exists());
3030 assert!(source.candidates[0].worktree.exists());
3031 let audit: serde_json::Value =
3032 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3033 assert_eq!(audit["version"], 2);
3034 assert_eq!(
3035 audit["owner"],
3036 serde_json::json!({"conversation": conversation.id})
3037 );
3038 dispatch(
3039 &queue,
3040 &f.home,
3041 &talks,
3042 &conversation.id,
3043 operation(Some(id)),
3044 )
3045 .await
3046 .unwrap();
3047 }
3048
3049 #[tokio::test]
3050 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3051 let f = Fx::new().await;
3052 let source = f.run("af00", true).await;
3053 let mut clean = f.run("af01", false).await;
3054 let mut dirty = f.run("af02", false).await;
3055 for state in [&mut clean, &mut dirty] {
3056 state.status = RunStatus::Prep;
3057 state.prep_complete = Some(false);
3058 state.candidates[0].empty = false;
3059 state.save_under(&f.home).unwrap();
3060 }
3061 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3062 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3063 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3064 let plan = reconciliation_plan(&f, &task, &source).await;
3065 let selected: Vec<_> = plan["candidates"]
3066 .as_array()
3067 .unwrap()
3068 .iter()
3069 .filter(|e| e["selected"] == true)
3070 .collect();
3071 assert_eq!(selected.len(), 1, "{plan:#}");
3072 assert_eq!(selected[0]["run"], clean.id);
3073 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3074 .await
3075 .unwrap();
3076 assert!(!clean.candidates[0].worktree.exists());
3077 assert!(dirty.candidates[0].worktree.exists());
3078 assert!(
3079 RunState::load_under(&dirty.id, &f.home)
3080 .unwrap()
3081 .retired_to
3082 .is_none()
3083 );
3084 }
3085
3086 #[tokio::test]
3087 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3088 let f = Fx::new().await;
3089 let mut source = f.run("aa00", true).await;
3090 let alternative = f.run("aa01", true).await;
3091 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3092 .await
3093 .unwrap();
3094 source.reviewed_commits = Some(vec![head.clone()]);
3095 source.save_under(&f.home).unwrap();
3096 let mut runs = vec![source.clone(), alternative.clone()];
3097 for i in 2..14 {
3098 runs.push(f.run(&format!("aa{i:02}"), false).await);
3099 }
3100 let mut task = f.task(&runs);
3101 task.pr_recovery = Some(crate::pr_recovery::Selection {
3102 url: "https://example.invalid/pull/1".into(),
3103 branch: "selected".into(),
3104 head: head.clone(),
3105 source_head: Some(head.clone()),
3106 source_run: Some(source.id.clone()),
3107 source_proof: None,
3108 disposition: Some(crate::pr_recovery::Disposition::Land),
3109 instruction: task.instruction.clone(),
3110 approved_at: Timestamp::now(),
3111 });
3112 source.pr_recovery = task.pr_recovery.clone();
3113 source.save_under(&f.home).unwrap();
3114 runs[0] = source.clone();
3115 crate::queue::Queue::at(f.home.join("queue"))
3116 .put(&mut task)
3117 .unwrap();
3118 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3119 let task_before = serde_json::to_value(&task).unwrap();
3120 let mut report = Report::default();
3121 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3122 assert_eq!(
3123 report.retired, 0,
3124 "automatic cleanup must still refuse blanket markers"
3125 );
3126 assert!(
3127 report
3128 .retained
3129 .iter()
3130 .any(|reason| reason.contains("explicitly retained"))
3131 );
3132 let mut total = 0;
3133 for expected in [5, 5, 2, 0] {
3134 let plan = reconciliation_plan(&f, &task, &source).await;
3135 let selected: Vec<_> = plan["candidates"]
3136 .as_array()
3137 .unwrap()
3138 .iter()
3139 .filter(|e| e["selected"] == true)
3140 .collect();
3141 assert_eq!(selected.len(), expected, "{plan:#}");
3142 for e in &selected {
3143 assert!(
3144 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3145 .unwrap()
3146 .retired_to
3147 .is_none()
3148 );
3149 }
3150 let operation = plan["id"].as_str().unwrap();
3151 for _ in 0..2 {
3152 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3153 .await
3154 .unwrap();
3155 }
3156 total += expected;
3157 }
3158 assert_eq!(total, 12);
3159 for run in &runs {
3160 let dir = f.home.join("runs").join(&run.id);
3161 assert!(dir.join("run.json").exists());
3162 assert!(dir.join("retain-recovery-sources").exists());
3163 assert!(
3164 git::branch_exists(&f.repo, &run.candidates[0].branch)
3165 .await
3166 .unwrap()
3167 );
3168 }
3169 assert!(source.candidates[0].worktree.exists());
3170 assert!(alternative.candidates[0].worktree.exists());
3171 assert_eq!(
3172 git::rev_parse(&f.repo, &source.candidates[0].branch)
3173 .await
3174 .unwrap(),
3175 head
3176 );
3177 assert_eq!(
3178 serde_json::to_value(
3179 crate::queue::Queue::at(f.home.join("queue"))
3180 .get(&task.id)
3181 .unwrap()
3182 )
3183 .unwrap(),
3184 task_before
3185 );
3186 assert_eq!(
3187 runs.iter()
3188 .filter(|r| RunState::load_under(&r.id, &f.home)
3189 .unwrap()
3190 .retired_to
3191 .is_none())
3192 .count(),
3193 2
3194 );
3195 }
3196
3197 #[tokio::test]
3198 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3199 let f = Fx::new().await;
3200 let source = f.run("ab00", true).await;
3201 let mut runs = vec![source.clone()];
3202 for i in 1..9 {
3203 runs.push(f.run(&format!("ab{i:02}"), false).await);
3204 }
3205 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3206 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3207 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3208 std::fs::write(
3209 runs[3].candidates[0].worktree.join("scratch/source"),
3210 "ignored source",
3211 )
3212 .unwrap();
3213 std::fs::write(
3214 runs[4].worktree_root().join("unknown"),
3215 "unregistered source",
3216 )
3217 .unwrap();
3218 let checkpoint = git::git(
3219 &runs[5].candidates[0].worktree,
3220 &[
3221 "rev-parse",
3222 "--path-format=absolute",
3223 "--git-path",
3224 "sequencer",
3225 ],
3226 )
3227 .await
3228 .unwrap();
3229 std::fs::create_dir(checkpoint.trim()).unwrap();
3230 runs[6].driver_exited = false;
3231 runs[6].driver_pid = Some(std::process::id());
3232 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3233 runs[6].save_under(&f.home).unwrap();
3234 runs[7].driver_exited = false;
3235 runs[7].driver_pid = None;
3236 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3238 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3239
3240 let hidden = &runs[8].candidates[0].worktree;
3241 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3242 .await
3243 .unwrap();
3244 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3245 assert!(
3246 git::git(hidden, &["status", "--porcelain"])
3247 .await
3248 .unwrap()
3249 .is_empty()
3250 );
3251 let task = f.task(&runs);
3252 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3253 let plan = reconciliation_plan(&f, &task, &source).await;
3254 assert!(
3255 plan["candidates"]
3256 .as_array()
3257 .unwrap()
3258 .iter()
3259 .all(|e| e["selected"] == false),
3260 "{plan:#}"
3261 );
3262 for run in &runs {
3263 assert!(run.candidates[0].worktree.exists());
3264 }
3265 }
3266
3267 #[tokio::test]
3268 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3269 let f = Fx::new().await;
3270 let source = f.run("ac00", true).await;
3271 let empty = f.run("ac01", false).await;
3272 let task = f.task(&[source.clone(), empty.clone()]);
3273 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3274 let plan = reconciliation_plan(&f, &task, &source).await;
3275 std::fs::write(
3276 f.home.join("runs").join(&source.id).join("new-report"),
3277 "new review checkpoint",
3278 )
3279 .unwrap();
3280 assert!(
3281 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3282 .await
3283 .is_err()
3284 );
3285 assert!(empty.candidates[0].worktree.exists());
3286 assert!(
3287 RunState::load_under(&empty.id, &f.home)
3288 .unwrap()
3289 .retired_to
3290 .is_none()
3291 );
3292 let marker = f
3293 .home
3294 .join("runs")
3295 .join(&empty.id)
3296 .join("retain-recovery-sources");
3297 std::fs::write(&marker, "different owner").unwrap();
3298 let plan = reconciliation_plan(&f, &task, &source).await;
3299 assert!(
3300 plan["candidates"]
3301 .as_array()
3302 .unwrap()
3303 .iter()
3304 .all(|e| e["selected"] == false)
3305 );
3306 }
3307
3308 #[tokio::test]
3309 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3310 for partial in [false, true] {
3311 let f = Fx::new().await;
3312 let source = f.run("ab10", true).await;
3313 let first = f.run("ab11", false).await;
3314 let second = f.run("ab12", false).await;
3315 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3316 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3317 let plan = reconciliation_plan(&f, &task, &source).await;
3318 let id = plan["id"].as_str().unwrap();
3319 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3320 std::fs::write(&staging, b"").unwrap();
3322 assert!(
3323 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3324 .await
3325 .is_err()
3326 );
3327 assert!(staging.exists());
3328 std::fs::remove_file(&staging).unwrap();
3329 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3330 .await
3331 .unwrap_err();
3332 assert!(
3333 error.to_string().contains("before run state rename"),
3334 "{error:#}"
3335 );
3336 assert!(
3337 RunState::load_under(&first.id, &f.home)
3338 .unwrap()
3339 .retired_to
3340 .is_some()
3341 );
3342 assert!(
3343 RunState::load_under(&second.id, &f.home)
3344 .unwrap()
3345 .retired_to
3346 .is_none()
3347 );
3348 assert!(first.candidates[0].worktree.exists());
3349 assert!(second.candidates[0].worktree.exists());
3350 let interrupted = std::fs::read(&staging).unwrap();
3351 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3353 assert!(
3354 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3355 .await
3356 .is_err()
3357 );
3358 assert_eq!(
3359 std::fs::read(&staging).unwrap(),
3360 b"unrelated recovery source"
3361 );
3362 std::fs::write(&staging, interrupted).unwrap();
3363 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3364 .await
3365 .unwrap();
3366 assert!(!staging.exists());
3367 assert!(!first.candidates[0].worktree.exists());
3368 assert!(!second.candidates[0].worktree.exists());
3369 assert_eq!(
3370 RunState::load_under(&second.id, &f.home)
3371 .unwrap()
3372 .retired_to,
3373 Some(source.id)
3374 );
3375 }
3376 }
3377
3378 #[tokio::test]
3379 async fn reconciliation_resumes_only_its_exact_durable_fence() {
3380 let f = Fx::new().await;
3381 let source = f.run("ad00", true).await;
3382 let empty = f.run("ad01", false).await;
3383 let task = f.task(&[source.clone(), empty.clone()]);
3384 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3385 let plan = reconciliation_plan(&f, &task, &source).await;
3386 let id = plan["id"].as_str().unwrap();
3387 assert!(
3388 reconcile::interrupt_after_fence(&f.home, &task, id)
3389 .await
3390 .is_err()
3391 );
3392 assert!(empty.candidates[0].worktree.exists());
3393 assert_eq!(
3394 RunState::load_under(&empty.id, &f.home)
3395 .unwrap()
3396 .retired_to
3397 .as_ref(),
3398 Some(&source.id)
3399 );
3400 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3401 .await
3402 .unwrap();
3403 assert!(!empty.candidates[0].worktree.exists());
3404 git::worktree_add_branch(
3406 &f.repo,
3407 &empty.candidates[0].worktree,
3408 "restore-empty",
3409 &f.base,
3410 )
3411 .await
3412 .unwrap();
3413 assert!(
3415 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3416 .await
3417 .is_err()
3418 );
3419 assert!(empty.candidates[0].worktree.exists());
3420 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3421 changed.prep_failure = Some("new owner edit".into());
3422 changed.save_under(&f.home).unwrap();
3423 assert!(
3424 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3425 .await
3426 .is_err()
3427 );
3428 }
3429
3430 #[tokio::test]
3431 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3432 let f = Fx::new().await;
3433 let source = f.run("ae00", true).await;
3434 let mut prep = f.run("ae01", false).await;
3435 let candidate = prep.candidates.pop().unwrap();
3436 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3437 prep.save_under(&f.home).unwrap();
3438 let reset = f.run("ae02", false).await;
3439 let wt = &reset.candidates[0].worktree;
3440 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3441 git::git(wt, &["add", "."]).await.unwrap();
3442 git::git(wt, &["commit", "-m", "unique implementation"])
3443 .await
3444 .unwrap();
3445 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3446 let unknown = f.run("ae03", false).await;
3447 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3448 let mut raw: serde_json::Value =
3449 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3450 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3451 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3452 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3453 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3454 let plan = reconciliation_plan(&f, &task, &source).await;
3455 let selected: Vec<_> = plan["candidates"]
3456 .as_array()
3457 .unwrap()
3458 .iter()
3459 .filter(|e| e["selected"] == true)
3460 .collect();
3461 assert_eq!(selected.len(), 1, "{plan:#}");
3462 assert_eq!(selected[0]["run"], prep.id);
3463 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3464 .await
3465 .unwrap();
3466 assert!(!candidate.worktree.exists());
3467 assert!(reset.candidates[0].worktree.exists());
3468 assert!(unknown.candidates[0].worktree.exists());
3469 }
3470
3471 #[tokio::test]
3472 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3473 let f = Fx::new().await;
3474 let mut state = f.run("aaaa", true).await;
3475 let candidate = state.candidates[0].worktree.clone();
3476 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3477 Fx::target(&candidate);
3478 for seat in ["review-1", "review-2"] {
3479 let wt = state.worktree_root().join(seat);
3480 git::worktree_add_detached(&f.repo, &wt, &f.base)
3481 .await
3482 .unwrap();
3483 Fx::target(&wt);
3484 }
3485 state.status = RunStatus::Landing;
3486 state.save_under(&f.home).unwrap();
3487 let second = Fx::new().await;
3488 let other = second.run("bbbb", true).await;
3489 let mut imported = other.clone();
3491 imported.save_under(&f.home).unwrap();
3492 Fx::target(&other.candidates[0].worktree);
3493 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3494 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3495 assert!(
3496 candidate.join("target/debug/output").exists(),
3497 "active writer protected"
3498 );
3499 assert!(
3500 !state
3501 .worktree_root()
3502 .join("review-1/target/debug/output")
3503 .exists()
3504 );
3505 assert!(
3506 !state
3507 .worktree_root()
3508 .join("review-2/target/debug/output")
3509 .exists()
3510 );
3511 assert!(
3512 !other.candidates[0]
3513 .worktree
3514 .join("target/debug/output")
3515 .exists()
3516 );
3517 assert!(out.freed_bytes >= 600);
3518 drop(guard);
3519 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3520 assert!(out.freed_bytes >= 200);
3521 assert_eq!(
3522 std::fs::read_to_string(candidate.join("source")).unwrap(),
3523 "uncommitted work"
3524 );
3525 assert!(
3526 git::branch_exists(&f.repo, &state.candidates[0].branch)
3527 .await
3528 .unwrap()
3529 );
3530 }
3531
3532 #[tokio::test]
3533 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3534 let f = Fx::new().await;
3535 let state = f.run("cc01", false).await;
3536 let wt = &state.candidates[0].worktree;
3537 Fx::target(wt);
3538 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3539 let dir = wt.join("target").join(dir);
3540 std::fs::create_dir_all(&dir).unwrap();
3541 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3542 }
3543 let manifest = wt.join("target/debug/output.d");
3545 let output = wt.join("target/debug/output");
3546 std::fs::write(
3547 &manifest,
3548 format!(
3549 "{}: {}\n",
3550 output.display(),
3551 wt.join("target/debug/notes.txt").display()
3552 ),
3553 )
3554 .unwrap();
3555 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3556 assert_eq!(report.freed_bytes, 200);
3557 assert!(!output.exists());
3558 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3559 assert_eq!(
3560 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3561 "user evidence"
3562 );
3563 }
3564 assert!(manifest.exists());
3566 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3567 std::fs::write(&output, "user note replacing an old binary").unwrap();
3568 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3569 assert_eq!(report.freed_bytes, 0);
3570 assert_eq!(
3571 std::fs::read_to_string(&output).unwrap(),
3572 "user note replacing an old binary"
3573 );
3574 }
3575
3576 #[tokio::test]
3577 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3578 let f = Fx::new().await;
3579 let state = f.run("cc02", false).await;
3580 let wt = &state.candidates[0].worktree;
3581 Fx::target(wt);
3582 let mut disk = Fx::disk();
3583 disk.worktree_limit_bytes = u64::MAX;
3584 let now = Fx::now();
3585 let first = sweep(&f.home, &disk, now).await;
3586 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3587 assert_eq!(
3588 sweep(&f.home, &disk, now).await.measured_bytes,
3589 first.measured_bytes
3590 );
3591 assert!(
3592 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3593 .await
3594 .measured_bytes
3595 > first.measured_bytes
3596 );
3597 disk.worktree_limit_bytes = 0;
3598 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3599 }
3600
3601 #[tokio::test]
3602 async fn unfinished_source_fold_reports_its_retention_reason() {
3603 let f = Fx::new().await;
3604 let mut source = f.run("cc03", true).await;
3605 let task = f.task(&[source.clone()]);
3606 let error = crate::graph::fold_run(&mut source, true, &f.home)
3607 .await
3608 .unwrap_err();
3609 assert!(error.to_string().contains(&task.id));
3610 assert!(source.candidates[0].worktree.exists());
3611 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3612 let error = crate::graph::fold_run(&mut source, true, &f.home)
3613 .await
3614 .unwrap_err();
3615 assert!(error.to_string().contains("explicitly retained"));
3616 }
3617
3618 #[tokio::test]
3619 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3620 let f = Fx::new().await;
3621 let mut source = f.run("cc04", true).await;
3622 let mut task = f.task(&[source.clone()]);
3623 task.hold_manual(Some("queue settlement pending".into()));
3624 let queue = crate::queue::Queue::at(f.home.join("queue"));
3625 queue.put(&mut task).unwrap();
3626 source.merge = Some(crate::run::MergeOutcome {
3627 mode: crate::config::MergeMode::Pr,
3628 ok: true,
3629 detail: "confirmed merged".into(),
3630 empty: false,
3631 });
3632 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3634 source.status = status;
3635 assert!(
3636 crate::graph::fold_run(&mut source, true, &f.home)
3637 .await
3638 .is_err()
3639 );
3640 assert!(source.candidates[0].worktree.exists());
3641 }
3642 git::git(
3643 &f.repo,
3644 &["merge", "--ff-only", &source.candidates[0].branch],
3645 )
3646 .await
3647 .unwrap();
3648 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3649 .await
3650 .unwrap();
3651 source.status = RunStatus::Merged;
3652 source.save_under(&f.home).unwrap();
3653 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3654 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3655 assert!(
3656 crate::graph::fold_run(&mut source, true, &f.home)
3657 .await
3658 .unwrap_err()
3659 .to_string()
3660 .contains("explicitly retained")
3661 );
3662 std::fs::remove_file(
3665 f.home
3666 .join("runs")
3667 .join(&source.id)
3668 .join("retain-recovery-sources"),
3669 )
3670 .unwrap();
3671 let (folded, unreadable) = crate::clean::fold_due(
3672 &f.home.join("runs"),
3673 &f.home,
3674 &f.bay,
3675 &Fx::disk(),
3676 Fx::now(),
3677 )
3678 .await
3679 .unwrap();
3680 assert_eq!((folded, unreadable), (1, 0));
3681 assert!(!source.candidates[0].worktree.exists());
3682 let evidence = f
3683 .home
3684 .join("runs")
3685 .join(&source.id)
3686 .join("artifacts/fold-evidence");
3687 let names: Vec<_> = std::fs::read_dir(evidence)
3688 .unwrap()
3689 .map(|entry| entry.unwrap().file_name())
3690 .collect();
3691 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3692 assert_eq!(
3693 queue.get(&task.id).unwrap().status,
3694 crate::queue::TaskStatus::Held
3695 );
3696 }
3697
3698 #[tokio::test]
3699 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3700 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3701 let f = Fx::new().await;
3702 let mut state = f.run("cc06", true).await;
3703 let candidate = state.candidates[0].clone();
3704 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3705 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3706 git::rescue_commit(&candidate.worktree, "second candidate change")
3707 .await
3708 .unwrap();
3709 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3710 state.tally = Some(crate::run::Tally {
3711 first_choice: Default::default(),
3712 borda: Default::default(),
3713 winner: 'A',
3714 rankings: 0,
3715 unanimous_initial: true,
3716 deliberated: false,
3717 changed_votes: 0,
3718 unanimous_final: true,
3719 tie_break: None,
3720 judges: 0,
3721 present: 0,
3722 quorum: 0,
3723 met_quorum: true,
3724 uncontested: Some("fixture".into()),
3725 });
3726 let reviewer = state.worktree_root().join("review-1");
3729 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3730 .await
3731 .unwrap();
3732 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3735 git::rescue_commit(&f.repo, "advance before squash")
3736 .await
3737 .unwrap();
3738 let published = if variant == "partial" {
3739 &intermediate
3740 } else {
3741 &head
3742 };
3743 git::git(&f.repo, &["merge", "--squash", published])
3744 .await
3745 .unwrap();
3746 git::git(&f.repo, &["commit", "-m", "squash candidate"])
3747 .await
3748 .unwrap();
3749 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3750 assert!(
3751 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3752 .await
3753 .unwrap()
3754 .ok()
3755 );
3756 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3757 git::rescue_commit(&f.repo, "advance after squash")
3758 .await
3759 .unwrap();
3760 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3761 .await
3762 .unwrap();
3763 if matches!(variant, "checkpoint" | "reviewer") {
3764 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3766 git::rescue_commit(&reviewer, "unpublished checkpoint")
3767 .await
3768 .unwrap();
3769 if variant == "checkpoint" {
3770 git::git(&reviewer, &["reset", "--hard", &intermediate])
3771 .await
3772 .unwrap();
3773 }
3774 }
3775 if variant == "loser" {
3776 let mut loser = candidate.clone();
3777 loser.label = 'B';
3778 loser.branch = "magi/cc06/B".into();
3779 loser.worktree = state.worktree_root().join("cand-B");
3780 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3781 .await
3782 .unwrap();
3783 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3784 git::rescue_commit(&loser.worktree, "independent loser")
3785 .await
3786 .unwrap();
3787 state.candidates.push(loser);
3788 }
3789 state.status = RunStatus::Merged;
3790 state.merge = Some(crate::run::MergeOutcome {
3791 mode: crate::config::MergeMode::Pr,
3792 ok: true,
3793 detail: "candidate A squash merged".into(),
3794 empty: false,
3795 });
3796 state.save_under(&f.home).unwrap();
3797 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3798 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3799 result.unwrap();
3800 assert!(
3801 !candidate.worktree.exists() && !reviewer.exists(),
3802 "{variant}"
3803 );
3804 assert!(
3805 !git::branch_exists(&f.repo, &candidate.branch)
3806 .await
3807 .unwrap()
3808 );
3809 let artifacts = f
3810 .home
3811 .join("runs")
3812 .join(&state.id)
3813 .join("artifacts/fold-evidence");
3814 assert_eq!(
3815 artifacts.join(format!("{head}.patch")).exists(),
3816 variant == "partial",
3817 "only a partially published winner needs evidence: {variant}"
3818 );
3819 assert_eq!(
3820 artifacts.join(format!("{intermediate}.patch")).exists(),
3821 variant == "partial",
3822 "squash proof also covers intermediate reviewer tips: {variant}"
3823 );
3824 let patches: Vec<_> = std::fs::read_dir(&artifacts)
3825 .unwrap()
3826 .map(|entry| entry.unwrap().path())
3827 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3828 .collect();
3829 let expected = match variant {
3830 "partial" => 2,
3831 "loser" | "reviewer" => 1,
3832 _ => 0,
3833 };
3834 assert_eq!(patches.len(), expected, "{variant}");
3835 if variant == "loser" {
3836 assert!(
3837 std::fs::read_to_string(&patches[0])
3838 .unwrap()
3839 .contains("losing candidate")
3840 );
3841 }
3842 assert!(artifacts.join("manifest.json").exists());
3843 if variant == "reviewer" {
3844 assert!(
3845 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3846 .unwrap()
3847 .contains("reviewer checkpoint")
3848 );
3849 }
3850 crate::graph::fold_run(&mut state, true, &f.home)
3851 .await
3852 .unwrap();
3853 }
3854 }
3855
3856 #[tokio::test]
3857 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3858 for missing in [true, false] {
3859 let f = Fx::new().await;
3860 let mut state = f.run("cc05", true).await;
3861 git::git(
3864 &f.repo,
3865 &["update-ref", "refs/remotes/origin/main", &f.base],
3866 )
3867 .await
3868 .unwrap();
3869 let candidate = state.candidates[0].clone();
3870 if !missing {
3871 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3872 git::rescue_commit(&f.repo, "advance base before rebase")
3873 .await
3874 .unwrap();
3875 git::git(&candidate.worktree, &["rebase", "main"])
3876 .await
3877 .unwrap();
3878 git::sync_to_head(&candidate.worktree).await.unwrap();
3881 }
3882 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3883 state.reviewed_commits = Some(Vec::new());
3885 state.candidates[0].worktree = state.worktree_root().join("under-review");
3886 git::git(
3887 &f.repo,
3888 &[
3889 "worktree",
3890 "move",
3891 candidate.worktree.to_str().unwrap(),
3892 state.candidates[0].worktree.to_str().unwrap(),
3893 ],
3894 )
3895 .await
3896 .unwrap();
3897 let reviewer = state.worktree_root().join("review-1");
3898 git::worktree_add_detached(&f.repo, &reviewer, &head)
3899 .await
3900 .unwrap();
3901 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3904 git::rescue_commit(&reviewer, "old review history")
3905 .await
3906 .unwrap();
3907 git::git(&reviewer, &["reset", "--hard", &head])
3908 .await
3909 .unwrap();
3910 if missing {
3911 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3912 }
3913 let run_dir = f.home.join("runs").join(&state.id);
3914 let artifacts = run_dir.join("artifacts");
3915 std::fs::create_dir_all(&artifacts).unwrap();
3916 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3917 if missing {
3918 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3919 }
3920 state.status = RunStatus::Merged;
3921 state.merge = Some(crate::run::MergeOutcome {
3922 mode: crate::config::MergeMode::Pr,
3923 ok: true,
3924 detail: "confirmed squash merge".into(),
3925 empty: false,
3926 });
3927 state.save_under(&f.home).unwrap();
3928 let owned = state.candidates[0].worktree.clone();
3930 state.candidates[0].worktree = f.home.join("missing-external");
3931 assert!(
3932 crate::graph::fold_run(&mut state, true, &f.home)
3933 .await
3934 .unwrap_err()
3935 .to_string()
3936 .contains("outside the inventoried")
3937 );
3938 state.candidates[0].worktree = owned;
3939 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
3941 assert!(
3942 crate::graph::fold_run(&mut state, true, &f.home)
3943 .await
3944 .is_err()
3945 );
3946 assert!(reviewer.exists());
3947 drop(guard);
3948 let evidence = artifacts.join("fold-evidence");
3950 std::fs::write(&evidence, "blocked").unwrap();
3951 assert!(
3952 crate::graph::fold_run(&mut state, true, &f.home)
3953 .await
3954 .is_err()
3955 );
3956 assert!(reviewer.exists());
3957 std::fs::remove_file(&evidence).unwrap();
3958 std::fs::create_dir_all(&evidence).unwrap();
3959 let oversized = evidence.join("capacity-fixture");
3960 std::fs::File::create(&oversized)
3961 .unwrap()
3962 .set_len(64 * 1024 * 1024)
3963 .unwrap();
3964 assert!(
3965 crate::graph::fold_run(&mut state, true, &f.home)
3966 .await
3967 .is_err()
3968 );
3969 assert!(reviewer.exists());
3970 std::fs::remove_file(oversized).unwrap();
3971 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
3974 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
3975 assert!(
3976 crate::graph::fold_run(&mut state, true, &f.home)
3977 .await
3978 .is_err()
3979 );
3980 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
3981 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
3982 .await
3983 .unwrap();
3984 std::fs::write(&checkpoint, &head).unwrap();
3985 assert!(
3986 crate::graph::fold_run(&mut state, true, &f.home)
3987 .await
3988 .is_err()
3989 );
3990 std::fs::remove_file(checkpoint).unwrap();
3991 let (folded, unreadable) = crate::clean::fold_due(
3992 &f.home.join("runs"),
3993 &f.home,
3994 &f.bay,
3995 &Fx::disk(),
3996 Fx::now(),
3997 )
3998 .await
3999 .unwrap();
4000 if folded != 1 {
4001 panic!(
4002 "normal cleanup failed: {:#}",
4003 crate::graph::fold_run(&mut state, true, &f.home)
4004 .await
4005 .unwrap_err()
4006 );
4007 }
4008 assert_eq!((folded, unreadable), (1, 0));
4009 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4010 state = RunState::load_under(&state.id, &f.home).unwrap();
4011 assert!(state.candidates[0].folded);
4012 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4013 crate::graph::fold_run(&mut state, true, &f.home)
4014 .await
4015 .unwrap();
4016 assert_eq!(
4017 std::fs::read(evidence.join("manifest.json")).unwrap(),
4018 manifest
4019 );
4020 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4021 .await
4022 .unwrap();
4023 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4024 assert!(
4025 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4026 .await
4027 .unwrap()
4028 .ok()
4029 );
4030 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4033 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4034 .as_str()
4035 .unwrap();
4036 let recovered = f.home.join("recovered");
4037 std::fs::create_dir_all(&recovered).unwrap();
4038 git::git(&recovered, &["init"]).await.unwrap();
4039 std::fs::write(
4040 recovered.join("source"),
4041 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4042 )
4043 .unwrap();
4044 git::git(
4045 &recovered,
4046 &[
4047 "apply",
4048 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4049 ],
4050 )
4051 .await
4052 .unwrap();
4053 assert_eq!(
4054 std::fs::read_to_string(recovered.join("source")).unwrap(),
4055 "cc05"
4056 );
4057 assert!(
4058 !std::fs::read(evidence.join(format!("{head}.patch")))
4059 .unwrap()
4060 .is_empty()
4061 );
4062 assert_eq!(
4063 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4064 "review findings"
4065 );
4066 if missing {
4067 assert_eq!(
4068 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4069 "original snapshot"
4070 );
4071 } else {
4072 assert!(!artifacts.join("cand-A.patch").exists());
4073 }
4074 assert!(run_dir.join("run.json").exists());
4075 }
4076 }
4077
4078 #[tokio::test]
4079 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4080 use std::os::unix::fs::PermissionsExt as _;
4081 let f = Fx::new().await;
4082 let source = f.run("dd03", true).await;
4083 let empty = f.run("dd04", false).await;
4084 let task = f.task(&[source, empty.clone()]);
4085 let entered = f.repo.join("proof-entered");
4086 let release = f.repo.join("proof-release");
4087 let hook = f.repo.join("proof-monitor");
4088 std::fs::write(&hook, format!(
4089 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4090 entered.display(), release.display())).unwrap();
4091 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4092 git::git(
4093 &f.repo,
4094 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4095 )
4096 .await
4097 .unwrap();
4098 struct ReleaseProof(PathBuf);
4099 impl Drop for ReleaseProof {
4100 fn drop(&mut self) {
4101 let _ = std::fs::write(&self.0, "continue");
4102 }
4103 }
4104 let _release_on_panic = ReleaseProof(release.clone());
4105 let home = f.home.clone();
4106 let sweep = tokio::spawn(async move {
4107 let mut report = Report::default();
4108 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4109 report
4110 });
4111 let watchdog = std::time::Duration::from_secs(60);
4114 tokio::time::timeout(watchdog, async {
4115 while !entered.exists() {
4116 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4117 }
4118 })
4119 .await
4120 .expect("Git retirement proof did not enter the blocking hook");
4121 let queue = crate::queue::Queue::at(f.home.join("queue"));
4122 let mut selected = queue.get(&task.id).unwrap();
4123 selected.resume_override = Some(crate::queue::OperatorResume {
4124 question_id: "during-slow-proof".into(),
4125 at: Fx::now(),
4126 conductor_rehold: None,
4127 forced: false,
4128 pinned_run: Some(empty.id.clone()),
4129 });
4130 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4131 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4135 Ok(result) => {
4136 result.unwrap().unwrap();
4137 true
4138 }
4139 Err(_) => {
4140 std::fs::write(&release, "continue").unwrap();
4141 update.await.unwrap().unwrap();
4142 false
4143 }
4144 };
4145 std::fs::write(release, "continue").unwrap();
4146 let report = tokio::time::timeout(watchdog, sweep)
4147 .await
4148 .expect("Git retirement proof did not finish after release")
4149 .unwrap();
4150 assert!(
4151 updated_while_blocked,
4152 "task writers must not wait behind Git retirement proof"
4153 );
4154 assert_eq!(report.retired, 0);
4155 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4156 assert!(empty.candidates[0].worktree.exists());
4157 assert!(
4158 RunState::load_under(&empty.id, &f.home)
4159 .unwrap()
4160 .retired_to
4161 .is_none()
4162 );
4163 }
4164
4165 #[tokio::test]
4166 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4167 let f = Fx::new().await;
4168 let source = f.run("dd01", true).await;
4169 let mut empty = f.run("dd02", false).await;
4170 let expected = f.task(&[source.clone(), empty.clone()]);
4171 let queue = crate::queue::Queue::at(f.home.join("queue"));
4172 let _claim = queue.claim(&expected.id).unwrap();
4173 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4175 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4176 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4177 let mut selected = queue.get(&expected.id).unwrap();
4178 selected.resume_override = Some(crate::queue::OperatorResume {
4179 question_id: "new-owner-decision".into(),
4180 at: Fx::now(),
4181 conductor_rehold: None,
4182 forced: false,
4183 pinned_run: Some(empty.id.clone()),
4184 });
4185 queue.put(&mut selected).unwrap();
4186 let error = commit_retirement(
4187 &f.home,
4188 &queue,
4189 &expected,
4190 &mut empty,
4191 &source.id,
4192 &expected.runs,
4193 Fx::now(),
4194 )
4195 .unwrap_err();
4196 assert!(error.to_string().contains("task changed"));
4197 assert!(
4198 RunState::load_under(&empty.id, &f.home)
4199 .unwrap()
4200 .retired_to
4201 .is_none()
4202 );
4203 assert!(empty.candidates[0].worktree.exists());
4204 assert_eq!(
4205 queue
4206 .get(&expected.id)
4207 .unwrap()
4208 .resume_override
4209 .unwrap()
4210 .pinned_run,
4211 Some(empty.id.clone())
4212 );
4213 assert!(
4214 !f.home
4215 .join("recovery-lineages")
4216 .join(format!("{}.json", expected.id))
4217 .exists()
4218 );
4219 assert!(
4221 commit_retirement(
4222 &f.home,
4223 &queue,
4224 &selected,
4225 &mut empty,
4226 &source.id,
4227 &selected.runs,
4228 Fx::now()
4229 )
4230 .unwrap_err()
4231 .to_string()
4232 .contains("selected this recovery source")
4233 );
4234 }
4235
4236 #[tokio::test]
4237 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4238 let f = Fx::new().await;
4239 let state = f.run("cccc", false).await;
4240 let wt = &state.candidates[0].worktree;
4241 Fx::target(wt);
4242 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4243 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4244 assert!(
4245 !declared_outputs(&wt.join("target"))
4246 .unwrap()
4247 .contains(&wt.join("target/debug/link"))
4248 );
4249 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4250 let cargo = cargo_locks(&wt.join("target")).unwrap();
4251 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4252 assert_eq!(out.freed_bytes, 0);
4253 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4254 drop(cargo);
4255 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4256 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4257 assert_eq!(out.freed_bytes, 0);
4258 drop(driver);
4259 let exclusive = OutputLease::take(wt, true).unwrap();
4260 assert!(
4261 OutputLease::borrow(wt).is_err(),
4262 "resume cannot start a writer during cleanup"
4263 );
4264 drop(exclusive);
4265 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4266 assert_eq!(
4267 std::fs::read(wt.join("target/notes")).unwrap(),
4268 b"user file"
4269 );
4270 }
4271
4272 #[tokio::test]
4273 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4274 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4275 let f = Fx::new().await;
4276 let source = f.run("aa10", true).await;
4277 let empty = f.run("aa11", false).await;
4278 f.task(&[source, empty.clone()]);
4279 let wt = &empty.candidates[0].worktree;
4280 Fx::target(wt);
4281 let path = wt.join("target").join(leftover);
4282 if leftover.ends_with(".log") {
4283 std::fs::write(&path, "keep log\n").unwrap();
4284 } else {
4285 std::fs::create_dir(&path).unwrap();
4287 }
4288 let mut disk = Fx::disk();
4289 disk.auto_fold = true;
4290 let report = sweep(&f.home, &disk, Fx::now()).await;
4291 assert_eq!(report.freed_bytes, 200, "{report:?}");
4292 assert_eq!(report.retired, 0, "{report:?}");
4293 assert!(!wt.join("target/debug/output").exists());
4294 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4295 assert!(path.exists());
4296 if leftover.ends_with(".log") {
4297 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4298 }
4299 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4300 assert!(worktrees(&empty).await.unwrap().contains(wt));
4301 assert!(
4302 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4303 .await
4304 .unwrap()
4305 );
4306 assert!(
4307 RunState::load_under(&empty.id, &f.home)
4308 .unwrap()
4309 .retired_to
4310 .is_none()
4311 );
4312 }
4313 }
4314
4315 #[tokio::test]
4316 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4317 let f = Fx::new().await;
4318 let state = f.run("aa12", false).await;
4319 let wt = &state.candidates[0].worktree;
4320 Fx::target(wt);
4321 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4322 assert_eq!(report.freed_bytes, 200, "{report:?}");
4323 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4324 for name in ["tmp", "magi-tools", "log"] {
4325 let path = wt.join("target").join(name);
4326 std::fs::create_dir(&path).unwrap();
4327 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4328 std::fs::remove_dir(path).unwrap();
4329 }
4330 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4331 }
4332
4333 #[tokio::test]
4334 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4335 for invalid in [
4336 "tag",
4337 "identity",
4338 "tracked",
4339 "profile-file",
4340 "profile-symlink",
4341 "identity-symlink",
4342 ] {
4343 let f = Fx::new().await;
4344 let state = f.run("aa13", false).await;
4345 let wt = &state.candidates[0].worktree;
4346 Fx::target(wt);
4347 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4348 let target = wt.join("target");
4349 match invalid {
4350 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4351 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4352 "tracked" => {
4353 git::git(wt, &["add", "-f", "target/debug/output"])
4354 .await
4355 .unwrap();
4356 }
4357 "profile-file" => {
4358 std::fs::write(target.join("release"), "keep").unwrap();
4359 }
4360 "profile-symlink" => {
4361 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4362 .unwrap();
4363 }
4364 "identity-symlink" => {
4365 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4366 .unwrap();
4367 std::os::unix::fs::symlink(
4368 target.join("saved-info"),
4369 target.join(".rustc_info.json"),
4370 )
4371 .unwrap();
4372 }
4373 _ => unreachable!(),
4374 }
4375 let before = std::fs::read(target.join("debug/output")).unwrap();
4376 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4377 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4378 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4379 assert!(target.join("tmp").is_dir());
4380 }
4381 }
4382
4383 #[tokio::test]
4384 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4385 let f = Fx::new().await;
4386 let source = f.run("0000", true).await;
4387 let empty = f.run("0001", false).await;
4388 let changed = f.run("0002", false).await;
4389 let unknown = f.run("0003", false).await;
4390 let unproven = f.run("0004", false).await;
4391 f.task(&[
4392 source.clone(),
4393 empty.clone(),
4394 changed.clone(),
4395 unknown.clone(),
4396 unproven.clone(),
4397 ]);
4398 for run in [&empty, &changed, &unknown] {
4399 Fx::target(&run.candidates[0].worktree);
4400 }
4401 let reviewer = empty.worktree_root().join("review-1");
4402 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4403 .await
4404 .unwrap();
4405 Fx::target(&reviewer);
4406 let mut disk = Fx::disk();
4407 disk.auto_fold = false;
4408 let report = sweep(&f.home, &disk, Fx::now()).await;
4409 assert_eq!(report.freed_bytes, 800, "{report:?}");
4410 let wt = &empty.candidates[0].worktree;
4411 assert!(wt.join("target/debug/output.d").exists());
4412 assert!(!wt.join("target/debug/output").exists());
4413 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4414 let proof: CargoResidue =
4415 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4416 assert_eq!(
4417 proof.files[Path::new("debug/output.d")],
4418 std::fs::read(wt.join("target/debug/output.d")).unwrap()
4419 );
4420 std::fs::write(
4421 changed.candidates[0].worktree.join("target/debug/output.d"),
4422 "owner notes",
4423 )
4424 .unwrap();
4425 std::fs::write(
4426 unknown.candidates[0].worktree.join("target/debug/notes"),
4427 "keep",
4428 )
4429 .unwrap();
4430 Fx::target(&unproven.candidates[0].worktree);
4431 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4432 disk.auto_fold = true;
4433 let locks = cargo_locks(&wt.join("target")).unwrap();
4434 let mut report = Report::default();
4435 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4436 assert_eq!(report.retired, 0, "{report:?}");
4437 assert!(
4438 report
4439 .retained
4440 .iter()
4441 .any(|reason| reason.contains("Cargo build"))
4442 );
4443 drop(locks);
4444 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4445 assert_eq!(report.retired, 1, "{report:?}");
4446 assert!(!wt.exists());
4447 assert!(!reviewer.exists());
4448 assert!(
4449 residue_proof(&f.home, &empty.id, &reviewer)
4450 .unwrap()
4451 .exists()
4452 );
4453 assert!(
4454 proof_path.exists(),
4455 "metadata evidence outlives the worktree"
4456 );
4457 assert_eq!(
4458 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4459 Some(source.id)
4460 );
4461 for run in [&changed, &unknown, &unproven] {
4462 assert!(run.candidates[0].worktree.exists());
4463 assert!(
4464 RunState::load_under(&run.id, &f.home)
4465 .unwrap()
4466 .retired_to
4467 .is_none()
4468 );
4469 }
4470 }
4471
4472 #[tokio::test]
4473 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4474 let f = Fx::new().await;
4475 let source = f.run("0000", true).await;
4476 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4477 .await
4478 .unwrap();
4479 let mut runs = vec![source.clone()];
4480 for n in 1..=12 {
4481 runs.push(f.run(&format!("{n:04}"), false).await);
4482 }
4483 let dirty = runs[1].candidates[0].worktree.clone();
4484 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4485 let ignored = runs[2].candidates[0].worktree.clone();
4486 std::fs::create_dir(ignored.join("scratch")).unwrap();
4487 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4488 runs[1].candidates.clear();
4491 runs[1].save_under(&f.home).unwrap();
4492 let task = f.task(&runs);
4493 let mut total = 0;
4494 for _ in 0..4 {
4495 let mut report = Report::default();
4496 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4497 assert!(report.retired <= 5);
4498 total += report.retired;
4499 }
4500 assert_eq!(total, 10);
4501 assert!(dirty.exists());
4502 assert!(ignored.exists());
4503 assert_eq!(
4504 git::rev_parse(&f.repo, &source.candidates[0].branch)
4505 .await
4506 .unwrap(),
4507 source_head
4508 );
4509 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4510 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4511 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4512 let mut released = task.clone();
4513 released.release();
4514 assert_eq!(
4515 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4516 RunState::load_under(id, &f.home)
4517 }),
4518 Some(source.id.clone())
4519 );
4520
4521 assert!(
4522 f.home
4523 .join("runs")
4524 .join(&retired.id)
4525 .join("run.json")
4526 .exists()
4527 );
4528 assert!(
4529 f.home
4530 .join("recovery-lineages")
4531 .join(format!("{}.json", task.id))
4532 .exists()
4533 );
4534 }
4535
4536 #[tokio::test]
4537 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4538 let f = Fx::new().await;
4539 let a = f.run("dddd", true).await;
4540 let b = f.run("eeee", true).await;
4541 let empty = f.run("ffff", false).await;
4542 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4543 assert!(
4544 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4545 "unfinished queue history protects older source branches from ordinary folding"
4546 );
4547 let mut prior = a.clone();
4548 assert!(
4549 crate::graph::fold_run(&mut prior, true, &f.home)
4550 .await
4551 .unwrap_err()
4552 .to_string()
4553 .contains(&task.id)
4554 );
4555 let mut report = Report::default();
4556 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4557 assert_eq!(report.retired, 0);
4558 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4559 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4560 Fx::target(&a.candidates[0].worktree);
4561 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4562 assert!(
4563 report.freed_bytes > 0,
4564 "source retention does not retain regenerable outputs"
4565 );
4566 assert!(a.candidates[0].worktree.exists());
4567 assert!(b.candidates[0].worktree.exists());
4568 assert!(empty.candidates[0].worktree.exists());
4569 }
4570
4571 #[tokio::test]
4572 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4573 let f = Fx::new().await;
4574 let authority = f.run("p000", true).await;
4575 let question_run = f.run("p001", false).await;
4576 let mut checkpoint = f.run("p002", false).await;
4577 checkpoint.land_approval = Some(crate::run::LandApproval {
4578 question: "pending".into(),
4579 head: f.base.clone(),
4580 });
4581 checkpoint.save_under(&f.home).unwrap();
4582 let retained = f.run("p003", false).await;
4583 std::fs::write(
4584 f.home
4585 .join("runs")
4586 .join(&retained.id)
4587 .join("retain-recovery-sources"),
4588 "owner",
4589 )
4590 .unwrap();
4591 let mut partial = f.run("p004", false).await;
4592 let reviewer = partial.worktree_root().join("review-1");
4593 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4594 .await
4595 .unwrap();
4596 partial.retired_to = Some(authority.id.clone());
4598 partial.save_under(&f.home).unwrap();
4599 assert!(
4600 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4601 .await
4602 .unwrap()
4603 );
4604 let git_checkpoint = f.run("p005", false).await;
4605 let location = git::git(
4606 &git_checkpoint.candidates[0].worktree,
4607 &[
4608 "rev-parse",
4609 "--path-format=absolute",
4610 "--git-path",
4611 "sequencer",
4612 ],
4613 )
4614 .await
4615 .unwrap();
4616 std::fs::create_dir(location.trim()).unwrap();
4617 let mut preservation = f.run("p006", false).await;
4618 preservation
4619 .candidate_preservation
4620 .insert('A', "pending preservation".into());
4621 preservation.save_under(&f.home).unwrap();
4622 let task = f.task(&[
4623 preservation.clone(),
4624 git_checkpoint.clone(),
4625 authority.clone(),
4626 question_run.clone(),
4627 checkpoint.clone(),
4628 retained.clone(),
4629 partial.clone(),
4630 ]);
4631 let questions = crate::ask::Questions::at(f.home.join("questions"));
4632 let mut question = crate::ask::Question::new(
4633 question_run.id.clone(),
4634 "implement".into(),
4635 "fixture".into(),
4636 "owner decision".into(),
4637 String::new(),
4638 vec!["keep".into()],
4639 );
4640 questions.put(&mut question).unwrap();
4641 let mut report = Report::default();
4642 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4643 assert_eq!(report.retired, 1, "{report:?}");
4644 assert!(!reviewer.exists());
4645 for state in [
4646 &question_run,
4647 &checkpoint,
4648 &retained,
4649 &git_checkpoint,
4650 &preservation,
4651 ] {
4652 assert!(state.candidates[0].worktree.exists());
4653 assert!(
4654 RunState::load_under(&state.id, &f.home)
4655 .unwrap()
4656 .retired_to
4657 .is_none()
4658 );
4659 }
4660 assert!(questions.get(&question.id).unwrap().status.open());
4661 assert_eq!(
4662 crate::queue::Queue::at(f.home.join("queue"))
4663 .get(&task.id)
4664 .unwrap()
4665 .status,
4666 task.status
4667 );
4668 }
4669
4670 #[tokio::test]
4671 async fn preservation_schema_migration_remains_distinct_from_retirement() {
4672 let f = Fx::new().await;
4673 let mut state = f.run("p007", false).await;
4674 state.schema = 15;
4675 state.save_under(&f.home).unwrap();
4676 let native = RunState::load_under(&state.id, &f.home).unwrap();
4677 assert_eq!(native.schema, crate::run::SCHEMA);
4678 assert!(native.candidate_preservation.is_empty());
4679 assert!(native.retired_to.is_none());
4680 state.schema = 14;
4681 state.save_under(&f.home).unwrap();
4682 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4683 assert!(legacy.candidate_preservation.contains_key(&'A'));
4684 legacy.save_under(&f.home).unwrap();
4685 let restored = RunState::load_under(&state.id, &f.home).unwrap();
4686 assert_eq!(
4687 restored.candidate_preservation,
4688 legacy.candidate_preservation
4689 );
4690 }
4691
4692 #[tokio::test]
4693 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4694 let f = Fx::new().await;
4695 let cache = f.repo.join("cache");
4696 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4697 std::fs::create_dir_all(&cache).unwrap();
4698 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4699 assert!(
4700 crate::cache::maintenance_prune(&f.home, &cache, 1)
4701 .unwrap()
4702 .is_none()
4703 );
4704 drop(guard);
4705 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4706 }
4707
4708 #[tokio::test]
4709 async fn launcher_death_does_not_release_live_tool_output_reservations() {
4710 let f = Fx::new().await;
4711 let run = f.run("abcd", false).await;
4712 let wt = &run.candidates[0].worktree;
4713 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4714 helper
4715 .quiet()
4716 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4717 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4718 .env("MAGI_TEST_WORKTREE", wt)
4719 .stdin(std::process::Stdio::null())
4720 .stdout(std::process::Stdio::null())
4721 .stderr(std::process::Stdio::null())
4722 .kill_on_drop(true);
4723 assert!(
4724 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4725 .await
4726 .unwrap()
4727 .unwrap()
4728 .success()
4729 );
4730 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4731 .unwrap()
4732 .trim()
4733 .parse()
4734 .unwrap();
4735 struct KillTool(i32);
4736 impl Drop for KillTool {
4737 fn drop(&mut self) {
4738 unsafe { libc::kill(self.0, libc::SIGKILL) };
4740 }
4741 }
4742 let tool = KillTool(pid);
4743 assert!(OutputLease::take(wt, true).is_err());
4744 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4745 drop(tool);
4746 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4747 loop {
4748 if OutputLease::take(wt, true).is_ok()
4749 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4750 {
4751 break;
4752 }
4753 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4754 }
4755 })
4756 .await
4757 .unwrap();
4758 }
4759
4760 #[tokio::test]
4761 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4762 let f = Fx::new().await;
4763 let run = f.run("abcd", false).await;
4764 let wt = &run.candidates[0].worktree;
4765 let cache = f.repo.join("cache");
4766 std::fs::create_dir(&cache).unwrap();
4767 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4768 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4769 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4770 cmd.quiet()
4771 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4772 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4773 .stdin(std::process::Stdio::null())
4774 .stdout(std::process::Stdio::null())
4775 .stderr(std::process::Stdio::null())
4776 .kill_on_drop(true);
4777 if clear_owner {
4778 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4779 } else {
4780 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4781 }
4782 let gib = 1024 * 1024 * 1024;
4783 let mut leases = vec![
4784 OutputLease::borrow(wt).unwrap().unwrap(),
4785 OutputLease::cache(&f.home, &cache, false).unwrap(),
4786 ];
4787 leases.extend(
4788 OutputLease::admit_with(
4789 &f.home,
4790 std::slice::from_ref(wt),
4791 8 * gib,
4792 |_| Ok(9 * gib),
4793 )
4794 .unwrap(),
4795 );
4796 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4797 assert!(
4798 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4799 .await
4800 .unwrap()
4801 .unwrap()
4802 .success()
4803 );
4804 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4805 struct KillTool(Option<i32>);
4806 impl Drop for KillTool {
4807 fn drop(&mut self) {
4808 if let Some(pid) = self.0 {
4810 unsafe { libc::kill(pid, libc::SIGKILL) };
4811 }
4812 }
4813 }
4814 let mut tool = KillTool(Some(pid));
4815 assert!(OutputLease::take(wt, true).is_err());
4819 let finish = if cancel {
4820 drop(running);
4821 None
4822 } else {
4823 Some(tokio::spawn(async move { running.finish().await }))
4824 };
4825 if clear_owner {
4826 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4827 assert!(crate::proc::pid_alive(pid as u32));
4828 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4829 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4830 assert!(
4831 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4832 9 * gib
4833 ))
4834 .is_err(),
4835 "orphan still owns its disk reservation"
4836 );
4837 assert!(
4838 crate::cache::maintenance_prune(&f.home, &cache, 0)
4839 .unwrap()
4840 .is_none()
4841 );
4842 if let Some(finish) = &finish {
4843 assert!(!finish.is_finished());
4844 }
4845 unsafe { libc::kill(pid, libc::SIGKILL) };
4847 }
4848 if let Some(finish) = finish {
4849 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4850 .await
4851 .unwrap()
4852 .unwrap();
4853 }
4854 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4855 loop {
4856 if OutputLease::take(wt, true).is_ok()
4857 && OutputLease::cache(&f.home, &cache, true).is_ok()
4858 && OutputLease::admit_with(
4859 &f.home,
4860 std::slice::from_ref(wt),
4861 8 * gib,
4862 |_| Ok(9 * gib),
4863 )
4864 .is_ok()
4865 {
4866 if !clear_owner {
4868 assert!(!crate::proc::pid_alive(pid as u32));
4869 }
4870 break;
4871 }
4872 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4873 }
4874 })
4875 .await
4876 .unwrap();
4877 tool.0 = None;
4878 }
4879 }
4880
4881 #[tokio::test]
4882 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4883 let f = Fx::new().await;
4884 let run = f.run("abcd", false).await;
4885 let wt = &run.candidates[0].worktree;
4886 let cache = f.repo.join("cache");
4887 std::fs::create_dir(&cache).unwrap();
4888 let pid_file = f.repo.join("child-pid");
4889 let mut cmd = tokio::process::Command::new("sh");
4890 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4891 .quiet()
4892 .env("CHILD_PID_FILE", &pid_file)
4893 .process_group(0)
4894 .kill_on_drop(true);
4895 let running = crate::seat_process::Running::spawn(
4896 cmd,
4897 None,
4898 vec![
4899 OutputLease::borrow(wt).unwrap().unwrap(),
4900 OutputLease::cache(&f.home, &cache, false).unwrap(),
4901 ],
4902 )
4903 .unwrap();
4904 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4905 while !pid_file.exists() {
4906 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4907 }
4908 })
4909 .await
4910 .unwrap();
4911 assert!(OutputLease::take(wt, true).is_err());
4912 assert!(
4913 crate::cache::maintenance_prune(&f.home, &cache, 0)
4914 .unwrap()
4915 .is_none()
4916 );
4917 let pid: u32 = std::fs::read_to_string(pid_file)
4918 .unwrap()
4919 .trim()
4920 .parse()
4921 .unwrap();
4922 drop(running);
4923 tokio::time::timeout(std::time::Duration::from_secs(10), async {
4924 loop {
4925 if let Ok(guard) = OutputLease::take(wt, true) {
4926 assert!(
4927 !crate::proc::pid_alive(pid),
4928 "output lock released before tool death"
4929 );
4930 drop(guard);
4931 break;
4932 }
4933 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4934 }
4935 })
4936 .await
4937 .unwrap();
4938 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4939 }
4940
4941 #[tokio::test]
4942 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
4943 let f = Fx::new().await;
4944 let reason = tokio::time::timeout(
4945 std::time::Duration::from_secs(2),
4946 emergency(vec![f.repo.clone()], u64::MAX),
4947 )
4948 .await
4949 .unwrap();
4950 assert!(reason.contains("disk pressure"));
4951 assert!(
4952 tokio::time::timeout(
4953 std::time::Duration::from_millis(10),
4954 emergency(vec![f.repo.clone()], 0)
4955 )
4956 .await
4957 .is_err()
4958 );
4959 assert_eq!(
4960 std::fs::read_to_string(f.repo.join("source")).unwrap(),
4961 "base\n"
4962 );
4963 }
4964
4965 #[tokio::test]
4966 async fn automatic_requeues_cannot_refund_failed_input_generation() {
4967 let f = Fx::new().await;
4968 let mut run = f.run("1234", false).await;
4969 run.prep_failure = Some("cherry-pick conflict".into());
4970 let mut task = f.task(std::slice::from_ref(&run));
4971 let inputs = crate::queue::recovery_inputs(&run.sources);
4972 for _ in 0..108 {
4973 task.recovery_failed(&run);
4974 task.requeue();
4975 assert_eq!(task.attempts, 0);
4976 }
4977 let bytes = serde_json::to_vec(&task).unwrap();
4978 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
4979 assert!(
4980 restored
4981 .recovery_exhausted(&run.base_commit, &inputs, 2)
4982 .is_some()
4983 );
4984 assert!(
4985 restored
4986 .recovery_exhausted("different-base", &inputs, 2)
4987 .is_none()
4988 );
4989 let original_base = run.base_commit.clone();
4990 run.base_commit = "different-base".into();
4991 restored.recovery_failed(&run);
4992 assert!(
4993 restored
4994 .recovery_exhausted(&original_base, &inputs, 2)
4995 .is_some(),
4996 "returning to old inputs does not replenish their budget"
4997 );
4998 assert!(
4999 restored
5000 .recovery_exhausted(&run.base_commit, &inputs, 2)
5001 .is_none()
5002 );
5003 restored.release_operator("test-owner", None);
5004 assert!(
5005 restored
5006 .recovery_exhausted(&run.base_commit, &inputs, 2)
5007 .is_none()
5008 );
5009 }
5010 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5011 let mut state = f.run(tag, useful).await;
5012 state.status = RunStatus::Superseded;
5013 state.save_under(&f.home).unwrap();
5014 state
5015 }
5016
5017 fn historical_review(state: &mut RunState, head: &str) {
5018 state.reviews.push(
5019 serde_json::from_value(serde_json::json!({
5020 "round": state.reviews.len() + 1, "head": head, "reviews": []
5021 }))
5022 .unwrap(),
5023 );
5024 }
5025
5026 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5027 let path = state.worktree_root().join("review-1");
5028 git::worktree_add_detached(&f.repo, &path, head)
5029 .await
5030 .unwrap();
5031 path
5032 }
5033
5034 #[tokio::test]
5035 async fn source_eviction_separates_run_authority_from_read_seats() {
5036 for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5037 let f = Fx::new().await;
5038 let mut state = inactive_source(&f, "ef01", true).await;
5039 state.status = status;
5040 state.pr = Some(
5041 serde_json::from_value(serde_json::json!({
5042 "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5043 "checks":"green", "round":1, "rounds":1
5044 }))
5045 .unwrap(),
5046 );
5047 state.pr_head = Some(state.candidates[0].branch.clone());
5048 state.pr_merged_head = Some(f.base.clone());
5049 state.recovery = Some(
5050 serde_json::from_value(serde_json::json!({
5051 "from_run": "20260101-000000-abcd", "generation": 1,
5052 "mode": "full-review"
5053 }))
5054 .unwrap(),
5055 );
5056 state.pr_recovery = Some(
5057 serde_json::from_value(serde_json::json!({
5058 "url": "https://example.invalid/pull/1", "branch": "public",
5059 "head": f.base, "source_run": state.id,
5060 "instruction": "keep implementation", "approved_at": Fx::now()
5061 }))
5062 .unwrap(),
5063 );
5064 state.save_under(&f.home).unwrap();
5065 let review = reviewer(&f, &state, &f.base).await;
5066 let mut other = inactive_source(&f, "ef02", true).await;
5067 other.pr_recovery = state.pr_recovery.clone();
5068 other.save_under(&f.home).unwrap();
5069 let mut task = f.task(&[state.clone(), other]);
5070 task.pr_recovery = state.pr_recovery.clone();
5071 crate::queue::Queue::at(f.home.join("queue"))
5072 .put(&mut task)
5073 .unwrap();
5074 let run_path = f.home.join("runs").join(&state.id).join("run.json");
5075 let before = std::fs::read(&run_path).unwrap();
5076 let task_before = serde_json::to_value(&task).unwrap();
5077 let mut report = Report::default();
5078 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5079 assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5080 assert!(!review.exists());
5081 assert!(state.candidates[0].worktree.exists());
5082 assert!(
5083 git::branch_exists(&f.repo, &state.candidates[0].branch)
5084 .await
5085 .unwrap()
5086 );
5087 assert_eq!(std::fs::read(&run_path).unwrap(), before);
5088 assert_eq!(
5089 serde_json::to_value(
5090 crate::queue::Queue::at(f.home.join("queue"))
5091 .get(&task.id)
5092 .unwrap()
5093 )
5094 .unwrap(),
5095 task_before
5096 );
5097 }
5098 }
5099
5100 #[tokio::test]
5101 async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5102 for kind in ["own-path", "active-seat", "paused-session"] {
5103 let f = Fx::new().await;
5104 let mut state = inactive_source(&f, "ef03", true).await;
5105 let review = reviewer(&f, &state, &f.base).await;
5106 let expected = match kind {
5107 "own-path" => {
5108 state.attachments.push(review.join("source"));
5109 "run.attachments[0]"
5110 }
5111 "active-seat" => {
5112 state.active.insert(
5113 "review-1".into(),
5114 serde_json::from_value(serde_json::json!({
5115 "node":"review", "started_at":Fx::now(), "timeout_secs":60
5116 }))
5117 .unwrap(),
5118 );
5119 "continuation lease"
5120 }
5121 _ => {
5122 state.status = RunStatus::Blocked;
5123 state.parked = true;
5124 let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5125 seat.turns = 2;
5126 seat.captured_session = Some("saved-session".into());
5127 state.seats.insert("review-1".into(), seat);
5128 "resume contract"
5129 }
5130 };
5131 state.save_under(&f.home).unwrap();
5132 let mut report = Report::default();
5133 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5134 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5135 assert!(review.exists());
5136 assert!(
5137 report.retained.iter().any(|r| r.contains(expected)),
5138 "{kind}: {report:?}"
5139 );
5140 restore_execution_seats(&f.home, &state).await.unwrap();
5143 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5144 }
5145 }
5146
5147 #[tokio::test]
5148 async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5149 for kind in ["waiting", "lease", "historical", "unknown"] {
5150 let f = Fx::new().await;
5151 let state = inactive_source(&f, "ef06", true).await;
5152 let review = reviewer(&f, &state, &f.base).await;
5153 let store = crate::ask::Questions::at(f.home.join("questions"));
5154 let mut question = crate::ask::Question::new(
5155 "20260101-000000-ffff".into(),
5156 "review".into(),
5157 "review-1".into(),
5158 "fixture".into(),
5159 "fixture".into(),
5160 Vec::new(),
5161 );
5162 question.cwd = Some(review.to_string_lossy().into_owned());
5163 if kind != "waiting" {
5164 question.status = crate::ask::QuestionStatus::Abandoned;
5165 }
5166 store.put(&mut question).unwrap();
5167 if kind == "lease" {
5168 store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5169 }
5170 if kind == "unknown" {
5171 let path = store.path_of(&question.id);
5172 let mut raw: serde_json::Value =
5173 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5174 raw["future_cwd_authority"] = serde_json::Value::Null;
5175 std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5176 }
5177 let mut report = Report::default();
5178 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5179 assert_eq!(
5180 report.evicted_copies,
5181 usize::from(kind == "historical"),
5182 "{kind}: {report:?}"
5183 );
5184 assert_eq!(review.exists(), kind != "historical");
5185 if kind == "waiting" || kind == "lease" {
5186 assert!(
5187 report
5188 .retained
5189 .iter()
5190 .any(|r| r.contains("question") && r.contains("cwd")),
5191 "{report:?}"
5192 );
5193 }
5194 }
5195 }
5196
5197 #[tokio::test]
5198 async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5199 let f = Fx::new().await;
5200 let mut state = inactive_source(&f, "ef07", true).await;
5201 let review = reviewer(&f, &state, &f.base).await;
5202 let mut report = Report::default();
5203 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5204 assert_eq!(report.evicted_copies, 1, "{report:?}");
5205 state.config.worktree.setup = vec![crate::config::SetupStep {
5206 run: Some("printf preserved > source; exit 1".into()),
5207 ..Default::default()
5208 }];
5209 state.save_under(&f.home).unwrap();
5210 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5211 let marker = f
5212 .home
5213 .join("runs")
5214 .join(&state.id)
5215 .join("artifacts/restore-review-1.pending");
5216 assert!(marker.exists());
5217 assert_eq!(
5218 std::fs::read_to_string(review.join("source")).unwrap(),
5219 "preserved"
5220 );
5221 assert!(
5222 restore_execution_seats(&f.home, &state)
5223 .await
5224 .unwrap_err()
5225 .to_string()
5226 .contains("interrupted execution restoration")
5227 );
5228 let mut retry = Report::default();
5229 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5230 assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5231 assert!(
5232 retry
5233 .retained
5234 .iter()
5235 .any(|r| r.contains("restoration checkpoint")),
5236 "{retry:?}"
5237 );
5238 assert!(marker.exists());
5239 assert_eq!(
5240 std::fs::read_to_string(review.join("source")).unwrap(),
5241 "preserved"
5242 );
5243 }
5244
5245 #[tokio::test]
5246 async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5247 for kind in [
5248 "alternates",
5249 "promisor-config",
5250 "promisor-pack",
5251 "objects-symlink",
5252 ] {
5253 let f = Fx::new().await;
5254 let state = inactive_source(&f, "ef11", true).await;
5255 let review = reviewer(&f, &state, &f.base).await;
5256 let common = crate::repo_lock::common_dir(&f.repo)
5257 .await
5258 .unwrap()
5259 .unwrap();
5260 match kind {
5261 "alternates" => {
5262 let borrowed = f._temp.path().join("borrowed-objects");
5263 std::fs::create_dir(&borrowed).unwrap();
5264 std::fs::write(
5265 common.join("objects/info/alternates"),
5266 format!("{}\n", borrowed.display()),
5267 )
5268 .unwrap();
5269 }
5270 "promisor-config" => {
5271 git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5272 .await
5273 .unwrap();
5274 }
5275 "promisor-pack" => {
5276 std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5277 }
5278 _ => {
5279 let borrowed = f._temp.path().join("borrowed-objects");
5280 std::fs::rename(common.join("objects"), &borrowed).unwrap();
5281 std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5282 }
5283 }
5284 let mut report = Report::default();
5285 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5286 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5287 assert!(review.exists());
5288 assert!(
5289 report.retained.iter().any(|r| r.contains("source anchor")),
5290 "{kind}: {report:?}"
5291 );
5292 }
5293 }
5294
5295 #[tokio::test]
5296 async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5297 let mut f = Fx::new().await;
5298 crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5299 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5300 assert_eq!(
5301 git::git(&f.repo, &["rev-list", "--count", &f.base])
5302 .await
5303 .unwrap(),
5304 "5001"
5305 );
5306 let state = inactive_source(&f, "ef10", true).await;
5307 let review = reviewer(&f, &state, &f.base).await;
5308 std::fs::write(review.join("source"), "bounded review delta").unwrap();
5309 git::git(&review, &["add", "."]).await.unwrap();
5310 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5311 let head = git::git(
5312 &review,
5313 &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5314 )
5315 .await
5316 .unwrap();
5317 git::git(&review, &["checkout", "--detach", &head])
5318 .await
5319 .unwrap();
5320 let run_file = f.home.join("runs").join(&state.id).join("run.json");
5321 let run_before = std::fs::read(&run_file).unwrap();
5322 let mut report = Report::default();
5323 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5324 assert_eq!(report.evicted_copies, 1, "{report:?}");
5325 assert!(!review.exists());
5326 assert!(state.candidates[0].worktree.exists());
5327 assert!(
5328 git::branch_exists(&f.repo, &state.candidates[0].branch)
5329 .await
5330 .unwrap()
5331 );
5332 assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5333 let archive = f
5334 .home
5335 .join("runs")
5336 .join(&state.id)
5337 .join("artifacts/source-evidence")
5338 .join(format!("{head}-{}", f.base));
5339 let manifest: serde_json::Value =
5340 serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5341 assert_eq!(manifest["anchor"], f.base);
5342 assert!(
5343 std::fs::metadata(archive.join("objects.pack"))
5344 .unwrap()
5345 .len()
5346 < 16 * 1024
5347 );
5348 let pin = format!("refs/magi/source-anchors/{}", f.base);
5349 assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5350 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5351 .await
5352 .unwrap();
5353 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5354 assert!(
5355 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5356 .await
5357 .unwrap()
5358 .ok()
5359 );
5360 let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5361 .await
5362 .unwrap();
5363 for kind in ["missing", "moved", "symbolic", "symlink"] {
5365 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5366 .await
5367 .unwrap();
5368 match kind {
5369 "moved" => {
5370 git::git(&f.repo, &["update-ref", &pin, &other])
5371 .await
5372 .unwrap();
5373 }
5374 "symbolic" => {
5375 git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5376 .await
5377 .unwrap();
5378 }
5379 "symlink" => {
5380 let common = crate::repo_lock::common_dir(&f.repo)
5381 .await
5382 .unwrap()
5383 .unwrap();
5384 let target = f._temp.path().join("operator-anchor");
5385 std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5386 let file = common.join(&pin);
5387 std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5388 std::os::unix::fs::symlink(target, file).unwrap();
5389 }
5390 _ => {}
5391 }
5392 assert!(
5393 restore_execution_seats(&f.home, &state).await.is_err(),
5394 "{kind}"
5395 );
5396 assert!(!review.exists());
5397 assert!(
5398 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5399 .await
5400 .unwrap()
5401 .ok()
5402 );
5403 if kind == "symlink" {
5404 let common = crate::repo_lock::common_dir(&f.repo)
5405 .await
5406 .unwrap()
5407 .unwrap();
5408 std::fs::remove_file(common.join(&pin)).unwrap();
5409 } else {
5410 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5411 .await
5412 .unwrap();
5413 }
5414 git::git(&f.repo, &["update-ref", &pin, &f.base])
5415 .await
5416 .unwrap();
5417 }
5418 assert!(
5421 git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5422 .await
5423 .unwrap()
5424 );
5425 git::git(
5426 &f.repo,
5427 &[
5428 "update-ref",
5429 "-d",
5430 &format!("refs/heads/{}", state.candidates[0].branch),
5431 ],
5432 )
5433 .await
5434 .unwrap();
5435 git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5436 .await
5437 .unwrap();
5438 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5439 .await
5440 .unwrap();
5441 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5442 assert_eq!(
5443 git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5444 .await
5445 .unwrap(),
5446 pin
5447 );
5448 assert_eq!(
5449 git::git(&f.repo, &["rev-list", "--count", &f.base])
5450 .await
5451 .unwrap(),
5452 "5001"
5453 );
5454 restore_execution_seats(&f.home, &state).await.unwrap();
5455 assert_eq!(
5456 git::git(&review, &["rev-list", "--count", "HEAD"])
5457 .await
5458 .unwrap(),
5459 "5002"
5460 );
5461 assert_eq!(
5462 git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5463 .await
5464 .unwrap(),
5465 "false"
5466 );
5467 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5468 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5469 let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5470 assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5471 std::fs::remove_dir_all(&f.repo).unwrap();
5472 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5473 let inspection = f._temp.path().join("shared-history-inspection");
5474 restore_seat(&f.home, &state.id, &head, &inspection)
5475 .await
5476 .unwrap();
5477 assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5478 assert_eq!(
5479 git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5480 .await
5481 .unwrap(),
5482 "true"
5483 );
5484 assert_eq!(
5485 git::git(&inspection, &["log", "--format=%H"])
5486 .await
5487 .unwrap(),
5488 format!("{head}\n{}", f.base)
5489 );
5490 assert!(!inspection.join(".git/objects/info/alternates").exists());
5491 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5492 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5493 }
5494
5495 #[tokio::test]
5496 async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5497 let f = Fx::new().await;
5498 let state = inactive_source(&f, "ef08", true).await;
5499 let review = reviewer(&f, &state, &f.base).await;
5500 let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5501 let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5502 .await
5503 .unwrap();
5504 let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
5505 .await
5506 .unwrap();
5507 git::git(&review, &["checkout", "--detach", &head])
5508 .await
5509 .unwrap();
5510 let archive =
5511 crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
5512 let mut report = Report::default();
5513 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5514 assert_eq!(report.evicted_copies, 0, "{report:?}");
5515 assert!(review.exists());
5516 assert!(
5517 report
5518 .retained
5519 .iter()
5520 .any(|r| r.contains("incomplete archived commit ancestry")),
5521 "{report:?}"
5522 );
5523 assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
5525 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5526 .await
5527 .unwrap();
5528 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5529 for oid in [&parent, &head] {
5530 assert!(
5531 !git::git_raw(&f.repo, &["cat-file", "-e", oid])
5532 .await
5533 .unwrap()
5534 .ok()
5535 );
5536 }
5537 let patch = source_diff(&archive).await.unwrap();
5538 assert!(
5539 crate::fold_evidence::import_snapshot(&archive, &f.repo)
5540 .await
5541 .is_err()
5542 );
5543 assert!(
5544 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5545 .await
5546 .unwrap()
5547 .ok()
5548 );
5549 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5550 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5551 let inspection = f._temp.path().join("legacy-tree-inspection");
5552 restore_source(&archive, &inspection).await.unwrap();
5553 assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
5554 assert!(
5555 !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
5556 .await
5557 .unwrap()
5558 .ok()
5559 );
5560 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5561 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5562 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5563 assert!(state.candidates[0].worktree.exists());
5564 }
5565
5566 #[tokio::test]
5567 async fn source_eviction_retains_ancestry_above_expanded_budget() {
5568 let f = Fx::new().await;
5569 let state = inactive_source(&f, "ef09", true).await;
5570 let review = reviewer(&f, &state, &f.base).await;
5571 std::fs::File::create(review.join("ancestor-only"))
5572 .unwrap()
5573 .set_len(64 * 1024 * 1024 + 1)
5574 .unwrap();
5575 git::git(&review, &["add", "."]).await.unwrap();
5576 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5577 let parent = git::git(
5578 &review,
5579 &["commit-tree", &parent_tree, "-m", "large ancestor"],
5580 )
5581 .await
5582 .unwrap();
5583 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5584 git::git(&review, &["add", "."]).await.unwrap();
5585 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5586 let head = git::git(
5587 &review,
5588 &[
5589 "commit-tree",
5590 &tree,
5591 "-p",
5592 &parent,
5593 "-m",
5594 "small final tree",
5595 ],
5596 )
5597 .await
5598 .unwrap();
5599 git::git(&review, &["checkout", "--detach", &head])
5600 .await
5601 .unwrap();
5602 let mut report = Report::default();
5603 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5604 assert_eq!(report.evicted_copies, 0, "{report:?}");
5605 assert!(
5606 report
5607 .retained
5608 .iter()
5609 .any(|r| r.contains("expanded object limit exceeded")),
5610 "{report:?}"
5611 );
5612 assert!(review.exists());
5613 assert!(state.candidates[0].worktree.exists());
5614 }
5615
5616 #[tokio::test]
5617 async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
5618 let f = Fx::new().await;
5619 let state = inactive_source(&f, "ef04", true).await;
5620 let review = reviewer(&f, &state, &f.base).await;
5621 std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
5622 git::git(&review, &["add", "."]).await.unwrap();
5623 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5624 let parent = git::git(
5625 &review,
5626 &["commit-tree", &parent_tree, "-m", "unrelated parent"],
5627 )
5628 .await
5629 .unwrap();
5630 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5631 std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
5632 git::git(&review, &["add", "."]).await.unwrap();
5633 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5634 let head = git::git(
5635 &review,
5636 &[
5637 "commit-tree",
5638 &tree,
5639 "-p",
5640 &parent,
5641 "-m",
5642 "unrelated review checkpoint",
5643 ],
5644 )
5645 .await
5646 .unwrap();
5647 git::git(&review, &["checkout", "--detach", &head])
5648 .await
5649 .unwrap();
5650 let mut report = Report::default();
5651 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5652 assert_eq!(report.evicted_copies, 1, "{report:?}");
5653 let ledger = f
5654 .home
5655 .join("runs")
5656 .join(&state.id)
5657 .join("artifacts/eviction-review-1.json");
5658 let original = std::fs::read(&ledger).unwrap();
5659 let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
5660 assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
5661 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5662 .await
5663 .unwrap();
5664 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5665 assert!(
5666 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5667 .await
5668 .unwrap()
5669 .ok()
5670 );
5671 assert!(
5672 !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
5673 .await
5674 .unwrap()
5675 .ok()
5676 );
5677 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5678 assert!(
5679 seat_heads(&f.home, &state.id)
5680 .await
5681 .unwrap()
5682 .contains_key(&head)
5683 );
5684 restore_execution_seats(&f.home, &state).await.unwrap();
5685 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
5686 assert_eq!(
5687 git::git(&review, &["log", "--format=%H"]).await.unwrap(),
5688 format!("{head}\n{parent}")
5689 );
5690 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5691 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5692 let inspection = f._temp.path().join("complete-history-inspection");
5693 restore_seat(&f.home, &state.id, &head, &inspection)
5694 .await
5695 .unwrap();
5696 assert_eq!(
5697 git::git(&inspection, &["log", "--format=%H"])
5698 .await
5699 .unwrap(),
5700 format!("{head}\n{parent}")
5701 );
5702 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5703 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5704
5705 assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
5706 assert!(
5707 !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
5708 .await
5709 .unwrap()
5710 .ok()
5711 );
5712 assert_eq!(
5713 std::fs::read_to_string(review.join("source")).unwrap(),
5714 "unrelated review checkpoint"
5715 );
5716 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5717 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5718 assert_eq!(
5719 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5720 "keep"
5721 );
5722 std::fs::remove_file(review.join("operator-notes")).unwrap();
5723 restore_execution_seats(&f.home, &state).await.unwrap();
5724 let mut second = Report::default();
5725 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
5726 assert_eq!(second.evicted_copies, 1, "{second:?}");
5727 assert_eq!(std::fs::read(&ledger).unwrap(), original);
5728 assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
5729 let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
5732 legacy["version"] = 1.into();
5733 legacy.as_object_mut().unwrap().remove("base");
5734 std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
5735 let mut advanced = state.clone();
5736 advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5737 .await
5738 .unwrap();
5739 advanced.save_under(&f.home).unwrap();
5740 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5741 let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
5743 let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
5744 let patch = std::fs::read(archive.join("change.patch")).unwrap();
5745 std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
5746 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5747 assert!(!review.exists());
5748 std::fs::write(archive.join("change.patch"), patch).unwrap();
5749 std::fs::create_dir(&review).unwrap();
5750 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5751 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5752 assert_eq!(
5753 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5754 "keep"
5755 );
5756 }
5757
5758 #[tokio::test]
5759 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
5760 use std::os::unix::fs::PermissionsExt;
5761 let mut f = Fx::new().await;
5762 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
5763 git::git(&f.repo, &["add", "."]).await.unwrap();
5764 git::git(&f.repo, &["commit", "-m", "binary base"])
5765 .await
5766 .unwrap();
5767 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5768 let mut state = inactive_source(&f, "ec01", true).await;
5769 let candidate = state.candidates[0].worktree.clone();
5770 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5771 historical_review(&mut state, &earlier);
5772 let review = reviewer(&f, &state, &earlier).await;
5773 std::fs::remove_file(candidate.join("source")).unwrap();
5774 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5775 std::fs::write(candidate.join("script"), "run\n").unwrap();
5776 std::fs::set_permissions(
5777 candidate.join("script"),
5778 std::fs::Permissions::from_mode(0o755),
5779 )
5780 .unwrap();
5781 git::git(&candidate, &["add", "."]).await.unwrap();
5782 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5783 .await
5784 .unwrap();
5785 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5786 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5787 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5788 .await
5789 .unwrap();
5790 historical_review(&mut state, &head);
5791 historical_review(&mut state, &head);
5792 state.status = RunStatus::Superseded;
5793 state.save_under(&f.home).unwrap();
5794 let task = f.task(std::slice::from_ref(&state));
5795 let before = serde_json::to_value(&state).unwrap();
5796 git::git(&review, &["reset", "--hard", &head])
5797 .await
5798 .unwrap();
5799 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5800 assert_eq!(report.evicted_copies, 1, "{report:?}");
5801 assert!(!review.exists());
5802 assert!(candidate.exists(), "selected implementation survives");
5803 assert!(report.source_unlinked_logical_bytes > 0);
5804 assert!(report.source_unlinked_allocated_bytes.is_some());
5805 assert!(report.source_evidence_added_allocated_bytes.is_some());
5806 assert_eq!(
5807 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5808 before
5809 );
5810 assert_eq!(
5811 crate::queue::Queue::at(f.home.join("queue"))
5812 .get(&task.id)
5813 .unwrap()
5814 .runs,
5815 task.runs
5816 );
5817 let archive = f
5818 .home
5819 .join("runs")
5820 .join(&state.id)
5821 .join("artifacts/source-evidence")
5822 .join(format!("{head}-{}", f.base));
5823 let patch = source_diff(&archive).await.unwrap();
5824 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5825 assert!(String::from_utf8_lossy(&patch).contains("100755"));
5826 assert!(
5827 git::worktree_remove_clean(&f.repo, &candidate)
5828 .await
5829 .unwrap()
5830 );
5831 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5832 .await
5833 .unwrap();
5834 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5835 .await
5836 .unwrap();
5837 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5838 assert!(
5839 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5840 .await
5841 .unwrap()
5842 .ok()
5843 );
5844 std::fs::remove_dir_all(&f.repo).unwrap();
5846 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5847 let restored = f._temp.path().join("restored");
5848 restore_source(&archive, &restored).await.unwrap();
5849 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5850 assert_eq!(
5851 std::fs::read(restored.join("binary")).unwrap(),
5852 b"new\0binary\xfe"
5853 );
5854 assert!(!restored.join("source").exists());
5855 assert_ne!(
5856 std::fs::metadata(restored.join("script"))
5857 .unwrap()
5858 .permissions()
5859 .mode()
5860 & 0o111,
5861 0
5862 );
5863 assert!(restore_source(&archive, &restored).await.is_err());
5864 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5867 .await
5868 .unwrap();
5869 let patch_file = f._temp.path().join("saved.patch");
5870 std::fs::write(&patch_file, &patch).unwrap();
5871 git::git(
5872 &restored,
5873 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5874 )
5875 .await
5876 .unwrap();
5877 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5878 assert_eq!(
5879 std::fs::read(restored.join("binary")).unwrap(),
5880 b"new\0binary\xfe"
5881 );
5882 }
5883
5884 #[tokio::test]
5885 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5886 let f = Fx::new().await;
5887 let state = inactive_source(&f, "ec10", true).await;
5888 let review = reviewer(&f, &state, &f.base).await;
5889 let mut successor = inactive_source(&f, "ec11", true).await;
5890 successor.retired_to = Some(state.id.clone());
5891 successor.save_under(&f.home).unwrap();
5892 f.task(&[state.clone(), successor]);
5893
5894 let mut tips = Vec::new();
5895 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5896 std::fs::write(review.join("source"), contents).unwrap();
5897 git::git(&review, &["add", "."]).await.unwrap();
5898 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5899 let tip = git::git(
5900 &review,
5901 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5902 )
5903 .await
5904 .unwrap();
5905 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5906 tips.push(tip);
5907 git::git(&review, &["reset", "--hard", &f.base])
5908 .await
5909 .unwrap();
5910 }
5911 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5913 .await
5914 .unwrap();
5915 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5916 .await
5917 .unwrap();
5918 let mut report = Report::default();
5919 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5920 assert_eq!(report.evicted_copies, 1, "{report:?}");
5921 assert!(!review.exists());
5922 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5923 .await
5924 .unwrap();
5925 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5926 for tip in &tips {
5927 assert!(
5928 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5929 .await
5930 .unwrap()
5931 .ok()
5932 );
5933 }
5934 std::fs::remove_dir_all(&f.repo).unwrap();
5936 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
5937 for (i, tip) in tips.iter().enumerate() {
5938 assert!(
5939 shown
5940 .iter()
5941 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
5942 );
5943 let destination = f._temp.path().join(format!("restore-history-{i}"));
5944 restore_seat(&f.home, &state.id, tip, &destination)
5945 .await
5946 .unwrap();
5947 assert_eq!(
5948 std::fs::read_to_string(destination.join("source")).unwrap(),
5949 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
5950 );
5951 }
5952 assert!(
5953 restore_seat(
5954 &f.home,
5955 &state.id,
5956 "unknown",
5957 &f._temp.path().join("missing")
5958 )
5959 .await
5960 .is_err()
5961 );
5962 }
5963
5964 #[tokio::test]
5965 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
5966 fn corrupt(path: &Path) -> Result<()> {
5967 let root = path.parent().unwrap();
5968 let marker = root.join("archive-to-corrupt");
5971 let archive = std::fs::read_to_string(&marker)?;
5972 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
5973 Ok(())
5974 }
5975 let f = Fx::new().await;
5976 let state = inactive_source(&f, "ec12", true).await;
5977 let review = reviewer(&f, &state, &f.base).await;
5978 let archive = f
5979 .home
5980 .join("runs")
5981 .join(&state.id)
5982 .join("artifacts/source-evidence")
5983 .join(format!("{}-{}", f.base, f.base));
5984 std::fs::write(
5985 state.worktree_root().join("archive-to-corrupt"),
5986 archive.to_str().unwrap(),
5987 )
5988 .unwrap();
5989 let mut report = Report::default();
5990 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
5991 assert_eq!(report.evicted_copies, 0, "{report:?}");
5992 assert!(review.exists());
5993 assert!(
5994 report
5995 .retained
5996 .iter()
5997 .any(|r| r.contains("corrupt snapshot")),
5998 "{report:?}"
5999 );
6000 }
6001
6002 #[tokio::test]
6003 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6004 for kind in [
6005 "dirty",
6006 "hidden",
6007 "ignored",
6008 "empty-directory",
6009 "untracked",
6010 "borrowed",
6011 "borrowed-symlink",
6012 "lease",
6013 "owner",
6014 "driver",
6015 "task-pin",
6016 "task-replacement",
6017 "malformed-orig-head",
6018 "grace",
6019 "resumable",
6020 "archive-symlink",
6021 "root-symlink",
6022 "checkpoint",
6023 "symlink",
6024 "retained",
6025 "unreadable",
6026 "unknown-field",
6027 "unknown-task-field",
6028 ] {
6029 let f = Fx::new().await;
6030 let mut state = inactive_source(&f, "ec02", true).await;
6031 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6032 .await
6033 .unwrap();
6034 historical_review(&mut state, &head);
6035 let review = reviewer(&f, &state, &head).await;
6036 let mut output = None;
6037 let mut owner = None;
6038 let mut driver = None;
6039 let mut disk = Fx::disk();
6040 match kind {
6041 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6042 "hidden" => {
6043 git::git(&review, &["update-index", "--assume-unchanged", "source"])
6044 .await
6045 .unwrap();
6046 std::fs::write(review.join("source"), "notes").unwrap();
6047 }
6048 "ignored" => {
6049 std::fs::create_dir(review.join("scratch")).unwrap();
6050 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6051 }
6052 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6053 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6054 "borrowed" => {
6055 let mut other = f.run("ec03", true).await;
6056 other.candidates[0].worktree = review.clone();
6057 other.save_under(&f.home).unwrap();
6058 }
6059 "borrowed-symlink" => {
6060 let alias = f._temp.path().join("borrowed-alias");
6061 std::os::unix::fs::symlink(&review, &alias).unwrap();
6062 let mut other = f.run("ec03", true).await;
6063 other.candidates[0].worktree = alias;
6064 other.save_under(&f.home).unwrap();
6065 }
6066 "lease" => output = OutputLease::borrow(&review).unwrap(),
6067 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6068 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6069 "task-pin" => {
6070 let mut task = f.task(std::slice::from_ref(&state));
6071 task.resume_override = Some(crate::queue::OperatorResume {
6072 question_id: "fixture-pin".into(),
6073 at: Fx::now(),
6074 conductor_rehold: None,
6075 forced: true,
6076 pinned_run: Some(state.id.clone()),
6077 });
6078 crate::queue::Queue::at(f.home.join("queue"))
6079 .put(&mut task)
6080 .unwrap();
6081 }
6082 "task-replacement" => {
6083 let mut task = f.task(std::slice::from_ref(&state));
6084 task.replacement = Some(crate::merge_decision::Replacement {
6085 question: "fixture-replacement".into(),
6086 run: state.id.clone(),
6087 pr: "https://example.invalid/pull/1".into(),
6088 agent: "worker".into(),
6089 context: String::new(),
6090 });
6091 crate::queue::Queue::at(f.home.join("queue"))
6092 .put(&mut task)
6093 .unwrap();
6094 }
6095 "malformed-orig-head" => {
6096 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6097 .await
6098 .unwrap();
6099 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6100 .unwrap();
6101 }
6102 "grace" => disk.fold_grace_secs = 3600,
6103 "resumable" => state.status = RunStatus::Blocked,
6104 "archive-symlink" => {
6105 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6106 std::fs::create_dir_all(&artifacts).unwrap();
6107 let outside = f._temp.path().join("outside");
6108 std::fs::create_dir(&outside).unwrap();
6109 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6110 .unwrap();
6111 }
6112 "root-symlink" => {
6113 let root = state.worktree_root();
6114 let outside = f._temp.path().join("outside-root");
6115 std::fs::rename(&root, &outside).unwrap();
6116 std::os::unix::fs::symlink(&outside, &root).unwrap();
6117 }
6118 "checkpoint" => {
6119 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6120 .await
6121 .unwrap();
6122 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6123 }
6124 "symlink" => {
6125 std::fs::remove_file(review.join("source")).unwrap();
6126 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6127 }
6128 "retained" => std::fs::write(
6129 f.home
6130 .join("runs")
6131 .join(&state.id)
6132 .join("retain-recovery-sources"),
6133 "owner",
6134 )
6135 .unwrap(),
6136 "unreadable" => {
6137 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6138 }
6139 "unknown-field" | "unknown-task-field" => {}
6140 _ => unreachable!(),
6141 }
6142 state.save_under(&f.home).unwrap();
6143 if kind == "unknown-field" {
6144 let path = f.home.join("runs").join(&state.id).join("run.json");
6145 let mut raw: serde_json::Value =
6146 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6147 raw["future_source_owner"] = serde_json::json!({"path": review});
6148 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6149 }
6150 if kind == "unknown-task-field" {
6151 let task = f.task(std::slice::from_ref(&state));
6152 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6153 let mut raw: serde_json::Value =
6154 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6155 raw["future_source_owner"] = serde_json::json!({"path": review});
6156 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6157 }
6158 let mut report = Report::default();
6159 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6160 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6161 assert!(review.exists(), "{kind}: {report:?}");
6162 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6163 drop(output);
6164 drop(owner);
6165 drop(driver);
6166 }
6167 }
6168
6169 #[tokio::test]
6170 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6171 let f = Fx::new().await;
6172 let state = inactive_source(&f, "ec04", true).await;
6173 let review = reviewer(&f, &state, &f.base).await;
6174 let mut report = Report::default();
6175 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6176 assert_eq!(report.evicted_copies, 1, "{report:?}");
6177 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6178 let ledger = artifacts.join("eviction-review-1.json");
6179 let mut record: serde_json::Value =
6180 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6181 record["phase"] = "pending".into();
6182 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6183 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6184 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6185 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6186 let mut blocked = Report::default();
6187 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6188 assert!(
6189 blocked
6190 .retained
6191 .iter()
6192 .any(|r| r.contains("corrupt snapshot")),
6193 "{blocked:?}"
6194 );
6195 assert_eq!(
6196 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6197 ["phase"],
6198 "pending"
6199 );
6200 assert!(
6201 restore_source(&archive, &f._temp.path().join("bad-restore"))
6202 .await
6203 .is_err()
6204 );
6205 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6206 let mut retried = Report::default();
6207 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6208 assert_eq!(
6209 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6210 ["phase"],
6211 "evicted"
6212 );
6213 assert_eq!(retried.evicted_copies, 0);
6214 assert!(!review.exists());
6215 let pending = archive.with_extension("pending");
6217 std::fs::rename(&archive, &pending).unwrap();
6218 assert_eq!(
6219 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6220 .await
6221 .unwrap(),
6222 archive
6223 );
6224 assert!(!pending.exists());
6225 }
6226
6227 #[tokio::test]
6228 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6229 let f = Fx::new().await;
6230 let mut empty = inactive_source(&f, "ec05", false).await;
6231 historical_review(&mut empty, &f.base);
6232 empty.status = RunStatus::Superseded;
6233 empty.save_under(&f.home).unwrap();
6234 let source = inactive_source(&f, "ec06", true).await;
6235 let task = f.task(&[empty.clone(), source.clone()]);
6236 let before = serde_json::to_value(&empty).unwrap();
6237 let mut report = Report::default();
6238 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6239 assert_eq!(report.evicted_copies, 1, "{report:?}");
6240 assert!(!empty.candidates[0].worktree.exists());
6241 assert!(source.candidates[0].worktree.exists());
6242 assert!(
6243 git::branch_exists(&f.repo, &empty.candidates[0].branch)
6244 .await
6245 .unwrap()
6246 );
6247 assert_eq!(
6248 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6249 before
6250 );
6251 let mut retry = Report::default();
6252 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6253 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6254 assert!(!empty.worktree_root().exists());
6255 assert_eq!(
6256 crate::queue::Queue::at(f.home.join("queue"))
6257 .get(&task.id)
6258 .unwrap()
6259 .runs,
6260 task.runs
6261 );
6262 }
6263 #[tokio::test]
6264 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6265 fn interrupt(_: &Path) -> Result<()> {
6266 anyhow::bail!("fixture interruption before unlink")
6267 }
6268 fn vanish(path: &Path) -> Result<()> {
6269 std::fs::remove_dir_all(path)?;
6270 anyhow::bail!("fixture interruption after checkout unlink")
6271 }
6272 for stage in [
6273 "intent",
6274 "missing-registration",
6275 "changed-admin",
6276 "missing-archive",
6277 "wrong-identity",
6278 ] {
6279 let f = Fx::new().await;
6280 let state = inactive_source(&f, "ec07", true).await;
6281 let review = reviewer(&f, &state, &f.base).await;
6282 let admin = PathBuf::from(
6283 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6284 .await
6285 .unwrap(),
6286 );
6287 let mut interrupted = Report::default();
6288 copies::evict_with(
6289 &f.home,
6290 &Fx::disk(),
6291 Fx::now(),
6292 &mut interrupted,
6293 if stage == "missing-registration" || stage == "changed-admin" {
6294 vanish
6295 } else {
6296 interrupt
6297 },
6298 )
6299 .await;
6300 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6301 let ledger = f
6302 .home
6303 .join("runs")
6304 .join(&state.id)
6305 .join("artifacts/eviction-review-1.json");
6306 let record: serde_json::Value =
6307 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6308 assert_eq!(record["phase"], "pending");
6309 if stage == "changed-admin" {
6310 std::fs::write(admin.join("owner-note"), "keep").unwrap();
6311 }
6312 if stage == "missing-archive" {
6313 std::fs::remove_file(
6314 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6315 )
6316 .unwrap();
6317 }
6318 if stage == "wrong-identity" {
6319 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6320 .await
6321 .unwrap();
6322 let other =
6323 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6324 .await
6325 .unwrap();
6326 let original = Path::new(record["evidence"].as_str().unwrap());
6327 for name in [
6328 "manifest.json",
6329 "objects.pack",
6330 "change.patch",
6331 "change.index",
6332 ] {
6333 std::fs::copy(other.join(name), original.join(name)).unwrap();
6334 }
6335 }
6336 let mut retried = Report::default();
6337 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6338 let phase =
6339 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6340 .unwrap()["phase"]
6341 .clone();
6342 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6343 assert_eq!(phase, "pending", "{stage}: {retried:?}");
6344 assert!(!retried.retained.is_empty());
6345 if stage == "changed-admin" {
6346 assert!(admin.join("owner-note").exists());
6347 } else {
6348 assert!(review.exists());
6349 }
6350 } else {
6351 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6352 assert!(!review.exists());
6353 assert!(!admin.exists());
6354 }
6355 }
6356 }
6357 #[tokio::test]
6358 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
6359 let f = Fx::new().await;
6360 let mut runs = Vec::new();
6361 let mut paths = Vec::new();
6362 for tag in ["ec10", "ec11", "ec12", "ec13"] {
6363 let state = inactive_source(&f, tag, true).await;
6364 for name in ["judge-1", "advisor-1"] {
6365 let path = state.worktree_root().join(name);
6366 git::worktree_add_detached(&f.repo, &path, &f.base)
6367 .await
6368 .unwrap();
6369 paths.push(path);
6370 }
6371 runs.push(state);
6372 }
6373 let task = f.task(&runs);
6374 let mut report = Report::default();
6375 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6376 assert_eq!(report.evicted_copies, 5, "{report:?}");
6377 let mut next = Report::default();
6378 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6379 assert_eq!(next.evicted_copies, 3, "{next:?}");
6380 assert!(paths.iter().all(|p| !p.exists()));
6381 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
6382 assert_eq!(
6383 crate::queue::Queue::at(f.home.join("queue"))
6384 .get(&task.id)
6385 .unwrap()
6386 .runs,
6387 task.runs
6388 );
6389 let limited = Fx::new().await;
6390 let state = inactive_source(&limited, "ec15", true).await;
6391 let copy = reviewer(&limited, &state, &limited.base).await;
6392 let evidence = limited
6393 .home
6394 .join("runs")
6395 .join(&state.id)
6396 .join("artifacts/source-evidence");
6397 std::fs::create_dir_all(&evidence).unwrap();
6398 let retained = evidence.join("reserved-evidence");
6399 std::fs::File::create(&retained)
6400 .unwrap()
6401 .set_len(64 * 1024 * 1024)
6402 .unwrap();
6403 let mut blocked = Report::default();
6404 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6405 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
6406 assert!(copy.exists());
6407 assert_eq!(
6408 std::fs::metadata(&retained).unwrap().len(),
6409 64 * 1024 * 1024
6410 );
6411 assert!(
6412 blocked
6413 .retained
6414 .iter()
6415 .any(|r| r.contains("per-run storage limit")),
6416 "{blocked:?}"
6417 );
6418 }
6419 #[tokio::test]
6420 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
6421 let f = Fx::new().await;
6422 let mut state = inactive_source(&f, "ec14", true).await;
6423 historical_review(&mut state, &f.base);
6424 historical_review(&mut state, &f.base);
6425 state.status = RunStatus::Superseded;
6426 state.save_under(&f.home).unwrap();
6427 let review = reviewer(&f, &state, &f.base).await;
6428 let task = f.task(std::slice::from_ref(&state));
6429 let path = f.home.join("runs").join(&state.id).join("run.json");
6430 let mut raw: serde_json::Value =
6431 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6432 raw["schema"] = 10.into();
6433 raw.as_object_mut().unwrap().remove("output_activity_at");
6434 raw["resource_wait"] = serde_json::Value::Null;
6437 raw["resource_block"] = serde_json::Value::Null;
6438 raw["driver_claimed"] = serde_json::Value::Null;
6439 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6440 let graph = raw["config"]["graph"].as_object_mut().unwrap();
6442 let count = graph.remove("implementers").unwrap();
6443 graph.insert("candidates".into(), count);
6444 let queue = crate::queue::Queue::at(f.home.join("queue"));
6445 let task_path = queue.path_of(&task.id);
6446 let mut task_raw: serde_json::Value =
6447 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
6448 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6449 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
6450 std::fs::write(&task_path, &task_historical).unwrap();
6451 let historical = serde_json::to_vec_pretty(&raw).unwrap();
6452 std::fs::write(&path, &historical).unwrap();
6453 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6454 assert_eq!(report.evicted_copies, 1, "{report:?}");
6455 assert!(!review.exists());
6456 assert_eq!(std::fs::read(&path).unwrap(), historical);
6457 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
6458 assert!(state.candidates[0].worktree.exists());
6459 assert_eq!(
6460 crate::queue::Queue::at(f.home.join("queue"))
6461 .get(&task.id)
6462 .unwrap()
6463 .runs,
6464 task.runs
6465 );
6466 }
6467 #[tokio::test]
6468 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
6469 let f = Fx::new().await;
6470 let state = inactive_source(&f, "ed01", true).await;
6471 let review = reviewer(&f, &state, &f.base).await;
6472 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
6473 git::git(&review, &["add", "."]).await.unwrap();
6474 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6475 let tip = git::git(
6476 &review,
6477 &[
6478 "commit-tree",
6479 &tree,
6480 "-p",
6481 &f.base,
6482 "-m",
6483 "temporary detached source",
6484 ],
6485 )
6486 .await
6487 .unwrap();
6488 git::git(&review, &["checkout", "--detach", &tip])
6489 .await
6490 .unwrap();
6491 git::git(&review, &["checkout", "--detach", &f.base])
6492 .await
6493 .unwrap();
6494 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
6495 .await
6496 .unwrap();
6497 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
6498 .await
6499 .unwrap();
6500 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
6501 .await
6502 .unwrap();
6503 assert!(!shown.lines().any(|oid| oid == tip));
6504 let admin = PathBuf::from(
6505 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6506 .await
6507 .unwrap(),
6508 );
6509 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
6510 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
6511 fn interrupt(_: &Path) -> Result<()> {
6512 anyhow::bail!("fixture interruption before unlink")
6513 }
6514 let mut pending = Report::default();
6515 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
6516 assert!(review.exists());
6517 let ledger = f
6518 .home
6519 .join("runs")
6520 .join(&state.id)
6521 .join("artifacts/eviction-review-1.json");
6522 let complete = std::fs::read(&ledger).unwrap();
6523 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
6524 incomplete["history"]
6525 .as_object_mut()
6526 .unwrap()
6527 .remove(&tip)
6528 .unwrap();
6529 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
6530 let mut blocked = Report::default();
6531 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6532 assert!(review.exists());
6533 assert!(
6534 blocked
6535 .retained
6536 .iter()
6537 .any(|reason| reason.contains("incomplete reflog checkpoint")),
6538 "{blocked:?}"
6539 );
6540 std::fs::write(&ledger, complete).unwrap();
6541 let mut report = Report::default();
6542 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6543 assert_eq!(report.evicted_copies, 1, "{report:?}");
6544 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6545 .await
6546 .unwrap();
6547 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6548 assert!(
6549 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
6550 .await
6551 .unwrap()
6552 .ok()
6553 );
6554 std::fs::remove_dir_all(&f.repo).unwrap();
6555 let destination = f._temp.path().join("restore-old-only");
6556 restore_seat(&f.home, &state.id, &tip, &destination)
6557 .await
6558 .unwrap();
6559 assert_eq!(
6560 std::fs::read(destination.join("old-only")).unwrap(),
6561 b"old-only\0binary\xff"
6562 );
6563 }
6564
6565 #[tokio::test]
6566 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
6567 fn interrupt(_: &Path) -> Result<()> {
6568 anyhow::bail!("fixture interruption before unlink")
6569 }
6570 fn vanish(path: &Path) -> Result<()> {
6571 std::fs::remove_dir_all(path)?;
6572 anyhow::bail!("fixture interruption before unregistration")
6573 }
6574 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
6575 let f = Fx::new().await;
6576 let mut state = inactive_source(&f, "ed02", true).await;
6577 state.driver_exited = false;
6578 state.driver_claimed = Some(true);
6579 state.driver_pid = Some(std::process::id());
6580 state.driver_started_at = crate::proc::process_started_at(std::process::id());
6581 state.begin_pass();
6582 state.status = RunStatus::Superseded;
6583 state.save_under(&f.home).unwrap();
6584 let review = reviewer(&f, &state, &f.base).await;
6585 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
6588 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
6589 std::fs::write(
6590 f.home.join("runs").join(&state.id).join("driver.json"),
6591 serde_json::to_vec(&identity).unwrap(),
6592 )
6593 .unwrap();
6594 let live = if stage == "live-driver" {
6595 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
6596 } else {
6597 None
6598 };
6599 assert_eq!(
6600 state.liveness_under(&f.home, false),
6601 if live.is_some() {
6602 Liveness::Live
6603 } else {
6604 Liveness::Dead
6605 }
6606 );
6607 let mut report = Report::default();
6608 if stage == "intent" || stage == "missing-registration" {
6609 copies::evict_with(
6610 &f.home,
6611 &Fx::disk(),
6612 Fx::now(),
6613 &mut report,
6614 if stage == "intent" { interrupt } else { vanish },
6615 )
6616 .await;
6617 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
6618 let mut retry = Report::default();
6619 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6620 assert!(!review.exists(), "{stage}: {retry:?}");
6621 let ledger = f
6622 .home
6623 .join("runs")
6624 .join(&state.id)
6625 .join("artifacts/eviction-review-1.json");
6626 let receipt: serde_json::Value =
6627 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
6628 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
6629 } else {
6630 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6631 assert_eq!(
6632 report.evicted_copies,
6633 usize::from(live.is_none()),
6634 "{stage}: {report:?}"
6635 );
6636 assert_eq!(review.exists(), live.is_some());
6637 }
6638 drop(live);
6639 assert!(
6640 !RunState::load_under(&state.id, &f.home)
6641 .unwrap()
6642 .driver_exited
6643 );
6644 }
6645 }
6646}