1mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7 let artifacts = home.join("runs").join(&state.id).join("artifacts");
8 if !artifacts.try_exists()? {
9 return Ok(false);
10 }
11 ensure!(
12 plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13 "noncanonical artifacts"
14 );
15 for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16 ensure!(i < 4096, "eviction record inspection limit exceeded");
17 let name = entry?.file_name();
18 let name = name.to_string_lossy();
19 if name.starts_with("eviction-") && name.ends_with(".json") {
20 return Ok(true);
21 }
22 }
23 Ok(false)
24}
25
26pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29 copies::restore_execution_seats(home, state).await
30}
31
32pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34 crate::fold_evidence::validate_snapshot(evidence).await?;
35 Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40 crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
45 copies::seat_evidence(home, id).await
46}
47
48pub async fn seat_heads(
50 home: &Path,
51 id: &str,
52) -> Result<std::collections::BTreeMap<String, String>> {
53 copies::seat_heads(home, id).await
54}
55
56pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
58 copies::seat_diff(home, id, head).await
59}
60
61pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
63 copies::restore_seat(home, id, head, destination).await
64}
65
66use crate::{
67 git,
68 proc::Quiet as _,
69 run::{Liveness, RunState},
70};
71use anyhow::{Context, Result, ensure};
72use jiff::Timestamp;
73use serde::{Deserialize, Serialize};
74use std::{
75 fs::{File, OpenOptions},
76 path::{Path, PathBuf},
77};
78
79#[derive(Debug)]
83pub(crate) struct OutputLease {
84 _file: std::sync::Arc<OutputFile>,
85 _activity: bool,
86}
87
88#[derive(Debug)]
89struct OutputFile(File);
90
91impl std::ops::Deref for OutputFile {
92 type Target = File;
93 fn deref(&self) -> &File {
94 &self.0
95 }
96}
97
98#[cfg(unix)]
99impl Drop for OutputFile {
100 fn drop(&mut self) {
101 use std::os::fd::AsRawFd;
102 unsafe {
104 libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
105 }
106 }
107}
108
109impl OutputLease {
110 #[cfg(unix)]
111 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
112 use std::os::fd::AsRawFd;
113 self._file.as_raw_fd()
114 }
115
116 pub(crate) fn pre_clean(dir: &Path) -> Result<Self> {
118 let file = OpenOptions::new()
119 .create(true)
120 .truncate(false)
121 .read(true)
122 .write(true)
123 .open(dir.join("magi-pre-clean.lock"))?;
124 crate::pr_head::try_lock_file(&file)
125 .context("pre-clean already running; retry after descendants terminate")?;
126 Ok(Self {
127 _file: std::sync::Arc::new(OutputFile(file)),
128 _activity: false,
129 })
130 }
131
132 pub(crate) fn duplicate(&self) -> Self {
133 Self {
134 _file: std::sync::Arc::clone(&self._file),
135 _activity: false,
136 }
137 }
138
139 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
140 if !worktree.join(".git").is_file() {
141 return Ok(None);
142 }
143 Self::take(worktree, false).map(Some)
144 }
145 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
146 use std::hash::{Hash, Hasher};
147 let mut hash = std::collections::hash_map::DefaultHasher::new();
148 let mut normalized = cache.to_owned();
149 let mut missing = Vec::new();
150 while !normalized.exists() {
151 let Some(name) = normalized.file_name() else {
152 break;
153 };
154 missing.push(name.to_owned());
155 if !normalized.pop() {
156 break;
157 }
158 }
159 normalized = normalized.canonicalize().unwrap_or(normalized);
160 for part in missing.into_iter().rev() {
161 normalized.push(part);
162 }
163 normalized.hash(&mut hash);
164 let dir = home.join("cache-leases");
165 std::fs::create_dir_all(&dir)?;
166 Self::lock(
167 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
168 exclusive,
169 )
170 }
171 #[cfg(unix)]
175 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
176 Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
177 }
178
179 #[cfg(unix)]
180 pub(crate) fn admit_with(
181 home: &Path,
182 paths: &[PathBuf],
183 min_free: u64,
184 free_bytes: impl Fn(&Path) -> Result<u64>,
185 ) -> Result<Vec<Self>> {
186 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
187 if min_free == 0 {
188 return Ok(Vec::new());
189 }
190 const RESERVE: u64 = 1024 * 1024 * 1024;
191 std::fs::create_dir_all(home)?;
192 let dir = home.canonicalize()?.join("disk-admission");
193 std::fs::create_dir_all(&dir)?;
194 ensure!(
195 plain_dir(&dir) && dir.canonicalize()? == dir,
196 "disk admission directory is not canonical"
197 );
198 let mut devices = std::collections::BTreeSet::new();
199 let mut reservations = Vec::new();
200 for path in paths {
201 let probe = path
202 .ancestors()
203 .find(|p| p.exists())
204 .context("disk admission path missing")?;
205 let device = std::fs::metadata(probe)?.dev();
206 if !devices.insert(device) {
207 continue;
208 }
209 let open = |path: &Path| -> Result<File> {
210 Ok(OpenOptions::new()
211 .read(true)
212 .write(true)
213 .create(true)
214 .truncate(false)
215 .custom_flags(libc::O_NOFOLLOW)
216 .open(path)?)
217 };
218 let admission = Self {
222 _file: std::sync::Arc::new(OutputFile(open(&dir.join(format!("{device}.lock")))?)),
223 _activity: false,
224 };
225 let started = std::time::Instant::now();
226 loop {
227 match crate::pr_head::try_lock_file(&admission._file) {
228 Ok(()) => break,
229 Err(error)
230 if matches!(
231 error.kind(),
232 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
233 ) && started.elapsed() < std::time::Duration::from_secs(1) =>
234 {
235 std::thread::sleep(std::time::Duration::from_millis(5));
236 }
237 Err(error) => {
238 return Err(error)
239 .context("disk admission busy or unavailable; retry without dispatch");
240 }
241 }
242 }
243 let mut active = 0u64;
244 let mut available = None;
245 for slot in 0..1024 {
246 let file = dir.join(format!("{device}-{slot}.reserve"));
247 if !file.exists() {
248 if available.is_none() {
249 available = Some(file);
250 }
251 continue;
252 }
253 let file = open(&file)?;
254 match crate::pr_head::try_lock_file(&file) {
255 Ok(()) if available.is_none() => {
256 available = Some(dir.join(format!("{device}-{slot}.reserve")))
257 }
258 Ok(()) => {}
259 Err(_) => active = active.saturating_add(RESERVE),
260 }
261 }
262 let free = free_bytes(probe)?;
263 let required = min_free.saturating_add(active).saturating_add(RESERVE);
264 ensure!(
265 free >= required,
266 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
267 );
268 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
269 crate::pr_head::try_lock_file(&file)?;
270 reservations.push(Self {
271 _file: std::sync::Arc::new(OutputFile(file)),
272 _activity: false,
273 });
274 }
275 Ok(reservations)
276 }
277
278 #[cfg(not(unix))]
279 pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
280 Ok(Vec::new())
282 }
283
284 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
285 let out = std::process::Command::new("git")
286 .args(["rev-parse", "--absolute-git-dir"])
287 .current_dir(worktree)
288 .quiet()
289 .output()?;
290 ensure!(
291 out.status.success(),
292 "cannot identify output lease directory"
293 );
294 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
295 Self::lock(&dir.join("magi-output.lock"), exclusive)
296 }
297 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
298 let file = OpenOptions::new()
299 .create(true)
300 .truncate(false)
301 .read(true)
302 .write(true)
303 .open(path)
304 .with_context(|| format!("output lease: open {}", path.display()))?;
305 #[cfg(unix)]
306 {
307 use std::os::fd::AsRawFd;
308 let mode = if exclusive {
309 libc::LOCK_EX
310 } else {
311 libc::LOCK_SH
312 };
313 ensure!(
315 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
316 "output lease: outputs are in use or being reclaimed: {}",
317 path.display()
318 );
319 }
320 #[cfg(not(unix))]
321 let _ = exclusive;
322 if !exclusive {
323 file.set_len(1)?;
325 file.set_modified(std::time::SystemTime::now())
326 .with_context(|| format!("record output activity {}", path.display()))?;
327 }
328 Ok(Self {
329 _file: std::sync::Arc::new(OutputFile(file)),
330 _activity: !exclusive,
331 })
332 }
333}
334#[cfg(unix)]
335impl Drop for OutputLease {
336 fn drop(&mut self) {
337 if self._activity
338 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
339 {
340 let _ = self._file.set_len(2);
343 tracing::warn!("could not record output lease release activity: {error}");
344 }
345 }
346}
347
348#[derive(Default, Debug, Serialize, Deserialize)]
349#[serde(default)]
350pub(crate) struct Report {
351 pub checked_at: Option<Timestamp>,
352 pub limit_bytes: u64,
353 pub reclaim_budget_bytes: u64,
354 pub measured_bytes: u64,
356 pub freed_bytes: u64,
358 pub measured_allocated_file_bytes: Option<u64>,
359 pub proven_reclaimable_bytes: u64,
361 pub proven_reclaimable_allocated_bytes: Option<u64>,
364 pub protected_or_unproven_bytes: u64,
365 pub estimated_released_allocated_bytes: Option<u64>,
368 pub observed_home_free_change_bytes: Option<i64>,
371 pub retired: usize,
372 pub evicted_copies: usize,
373 pub removed_empty_roots: usize,
374 pub source_unlinked_logical_bytes: u64,
375 pub source_unlinked_allocated_bytes: Option<u64>,
376 pub source_evidence_added_logical_bytes: u64,
377 pub source_evidence_added_allocated_bytes: Option<u64>,
378 pub retained: Vec<String>,
379 pub failures: Vec<String>,
380}
381
382#[cfg(unix)]
383fn allocated_inventory(
384 root: &Path,
385 seen: &mut std::collections::BTreeSet<(u64, u64)>,
386) -> Result<u64> {
387 use std::os::unix::fs::MetadataExt;
388 ensure!(
389 plain_dir(root) && root.canonicalize()? == root,
390 "allocated inventory root is not canonical"
391 );
392 let mut dirs = vec![root.to_owned()];
393 let mut bytes = 0u64;
394 while let Some(dir) = dirs.pop() {
395 for entry in std::fs::read_dir(dir)? {
396 let path = entry?.path();
397 let metadata = std::fs::symlink_metadata(&path)?;
398 if metadata.is_dir() {
399 dirs.push(path);
400 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
401 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
402 }
403 }
404 }
405 Ok(bytes)
406}
407
408#[derive(Default)]
409struct ReleasedAllocation {
410 #[cfg(unix)]
411 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
412 #[cfg(unix)]
413 removed: std::collections::BTreeMap<(u64, u64), u64>,
414}
415
416impl ReleasedAllocation {
417 fn of(paths: &[PathBuf]) -> Result<Self> {
418 #[cfg(unix)]
419 {
420 use std::os::unix::fs::MetadataExt;
421 let mut files = std::collections::BTreeMap::new();
422 for path in paths {
423 let m = std::fs::symlink_metadata(path)?;
424 ensure!(m.is_file(), "allocation candidate is not regular");
425 files.insert(
426 path.clone(),
427 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
428 );
429 }
430 Ok(Self {
431 files,
432 removed: Default::default(),
433 })
434 }
435 #[cfg(not(unix))]
436 {
437 let _ = paths;
438 Ok(Self::default())
439 }
440 }
441 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
442 #[cfg(unix)]
443 {
444 let mut inodes = std::collections::BTreeMap::new();
445 for &(device, inode, links, blocks) in self.files.values() {
446 let entry = inodes
447 .entry((device, inode))
448 .or_insert((0u64, links, blocks));
449 entry.0 += 1;
450 }
451 Some(
452 inodes
453 .values()
454 .filter(|(count, links, _)| count == links)
455 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
456 )
457 }
458 #[cfg(not(unix))]
459 {
460 None
461 }
462 }
463
464 fn unlinked(&mut self, path: &Path) -> Option<u64> {
465 #[cfg(unix)]
466 {
467 let (device, inode, links, blocks) = self.files[path];
468 let removed = self.removed.entry((device, inode)).or_default();
469 *removed += 1;
470 Some(if *removed == links { blocks } else { 0 })
471 }
472 #[cfg(not(unix))]
473 {
474 let _ = path;
475 None
476 }
477 }
478}
479
480fn plain_dir(path: &Path) -> bool {
481 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
482}
483
484fn run_paths_missing(state: &RunState) -> Result<bool> {
487 for path in [state.worktree_root(), state.repo.clone()] {
488 match std::fs::metadata(&path) {
489 Ok(_) => {}
490 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
491 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
492 }
493 }
494 Ok(false)
495}
496
497pub(crate) async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
499 let root = state.worktree_root();
500 if !plain_dir(&root) {
501 return Ok(Vec::new());
502 }
503 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
504 registered_worktrees(state, &list)
505}
506
507fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
508 let root = state.worktree_root();
509 if !plain_dir(&root) {
510 return Ok(Vec::new());
511 }
512 let root = root.canonicalize()?;
513 let mut paths = Vec::new();
514 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
515 let path = PathBuf::from(path);
516 #[cfg(windows)]
517 let path = if plain_dir(&path) {
518 git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
519 } else {
520 continue;
521 };
522 if plain_dir(&path)
523 && path.parent() == Some(root.as_path())
524 && path.canonicalize()? == path
525 && path.join(".git").is_file()
526 {
527 paths.push(path);
528 }
529 }
530 Ok(paths)
531}
532
533async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
536 validate_cargo_target(worktree, false).await
537}
538
539async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
540 let target = worktree.join("target");
541 ensure!(plain_dir(&target), "target absent or not a plain directory");
542 ensure!(target.canonicalize()? == target, "target path changed");
543 ensure!(
544 git::git(worktree, &["ls-files", "--", "target"])
545 .await?
546 .is_empty(),
547 "tracked target contents"
548 );
549 ensure!(
550 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
551 .await?
552 .ok(),
553 "target is not ignored"
554 );
555 cache_tag(&target, allow_missing_tag)?;
556 let path = target.join(".rustc_info.json");
557 plain_file(&path, "rustc identity")?;
558 let info: serde_json::Value = serde_json::from_slice(
559 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
560 )
561 .with_context(|| format!("parse rustc identity {}", path.display()))?;
562 ensure!(
563 info.get("rustc_fingerprint").is_some(),
564 "unknown Cargo identity {}",
565 path.display()
566 );
567 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
568 ensure!(
569 info.get("rustc_fingerprint")
570 .is_some_and(serde_json::Value::is_u64)
571 && info
572 .get("outputs")
573 .and_then(serde_json::Value::as_object)
574 .is_some_and(|outputs| outputs.values().any(|output| {
575 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
576 && output
577 .get("stdout")
578 .and_then(serde_json::Value::as_str)
579 .is_some_and(|stdout| stdout.starts_with("rustc "))
580 && output
581 .get("stderr")
582 .is_some_and(serde_json::Value::is_string)
583 }))
584 && info
585 .get("successes")
586 .is_some_and(serde_json::Value::is_object),
587 "missing-tag rustc identity structure invalid: {}",
588 path.display()
589 );
590 let mut profiles = 0;
591 for profile in ["debug", "release"] {
592 let dir = target.join(profile);
593 if !dir.exists() {
594 continue;
595 }
596 for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
597 ensure!(
598 plain_dir(&subdir) && subdir.canonicalize()? == subdir,
599 "missing-tag profile proof: {} is not a plain directory",
600 subdir.display()
601 );
602 }
603 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
604 profiles += 1;
605 }
606 ensure!(
607 profiles > 0,
608 "missing-tag proof: no Cargo profile in {}",
609 target.display()
610 );
611 }
612 for name in ["debug", "release"] {
613 let path = target.join(name);
614 match std::fs::symlink_metadata(&path) {
615 Ok(metadata) => ensure!(
616 metadata.is_dir() && path.canonicalize()? == path,
617 "profile validation: not a plain directory: {}",
618 path.display()
619 ),
620 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
621 Err(error) => {
622 return Err(error)
623 .with_context(|| format!("profile validation: inspect {}", path.display()));
624 }
625 }
626 }
627 Ok(target)
628}
629
630fn plain_file(path: &Path, stage: &str) -> Result<()> {
631 let meta = std::fs::symlink_metadata(path)
632 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
633 ensure!(
634 meta.is_file() && path.canonicalize()? == path,
635 "{stage}: not a plain file: {}",
636 path.display()
637 );
638 Ok(())
639}
640
641fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
642 let path = target.join("CACHEDIR.TAG");
643 match std::fs::symlink_metadata(&path) {
644 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
645 return Ok(());
646 }
647 result => {
648 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
649 }
650 }
651 plain_file(&path, "cache tag")?;
652 let tag = std::fs::read_to_string(&path)
653 .with_context(|| format!("cache tag: read {}", path.display()))?;
654 ensure!(
655 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
656 "unknown Cargo cache tag: {}",
657 path.display()
658 );
659 Ok(())
660}
661
662fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
666 let mut dirs = vec![target.to_owned()];
667 let mut entries = 0;
668 while let Some(dir) = dirs.pop() {
669 for entry in std::fs::read_dir(&dir)
670 .with_context(|| format!("output activity: list {}", dir.display()))?
671 {
672 let path = entry?.path();
673 entries += 1;
674 ensure!(
675 entries <= 100_000,
676 "output activity inventory too large: {}",
677 target.display()
678 );
679 let metadata = std::fs::symlink_metadata(&path)
680 .with_context(|| format!("output activity: inspect {}", path.display()))?;
681 if metadata.is_dir() {
682 if dir != target
684 || matches!(
685 path.file_name().and_then(|s| s.to_str()),
686 Some("debug" | "release")
687 )
688 {
689 dirs.push(path);
690 }
691 } else if metadata.is_file()
692 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
693 {
694 let at =
695 Timestamp::try_from(metadata.modified().with_context(|| {
696 format!("output activity: modified {}", path.display())
697 })?)?;
698 ensure!(
699 crate::clean::due(now, at, grace),
700 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
701 path.display()
702 );
703 }
704 }
705 }
706 Ok(())
707}
708
709pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
710 use std::io::Read as _;
711 let mut header = [0; 8];
712 let n = File::open(path)?.read(&mut header)?;
713 let header = &header[..n];
714 Ok(header.starts_with(b"\x7fELF")
715 || header.starts_with(b"!<arch>\n")
716 || header.starts_with(b"rust\0\0\0")
717 || header.starts_with(b"MZ\x90\0")
718 || [
719 [0xfe, 0xed, 0xfa, 0xce],
720 [0xce, 0xfa, 0xed, 0xfe],
721 [0xfe, 0xed, 0xfa, 0xcf],
722 [0xcf, 0xfa, 0xed, 0xfe],
723 [0xca, 0xfe, 0xba, 0xbe],
724 [0xca, 0xfe, 0xba, 0xbf],
725 ]
726 .iter()
727 .any(|magic| header.starts_with(magic)))
728}
729
730fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
734 let mut outputs = std::collections::BTreeSet::new();
735 for profile in ["debug", "release"] {
736 for dir in [target.join(profile), target.join(profile).join("deps")] {
737 if !plain_dir(&dir) || dir.canonicalize()? != dir {
738 continue;
739 }
740 for entry in std::fs::read_dir(&dir)? {
741 let path = entry?.path();
742 if path.extension().is_none_or(|ext| ext != "d")
743 || !std::fs::symlink_metadata(&path)?.is_file()
744 || std::fs::metadata(&path)?.len() > 1024 * 1024
745 || path.canonicalize()? != path
746 {
747 continue;
748 }
749 let body = std::fs::read_to_string(&path)?;
750 for line in body.lines() {
751 let Some((names, dependencies)) = line.split_once(": ") else {
752 continue;
753 };
754 if dependencies.trim().is_empty() {
755 continue;
756 }
757 let mut words = Vec::new();
760 let mut word = String::new();
761 let mut escaped = false;
762 for c in names.chars() {
763 if escaped {
764 word.push(c);
765 escaped = false;
766 } else if c == '\\' {
767 escaped = true;
768 } else if c.is_whitespace() {
769 if !word.is_empty() {
770 words.push(std::mem::take(&mut word));
771 }
772 } else {
773 word.push(c);
774 }
775 }
776 if escaped || word.contains(['$', '%']) {
777 continue;
778 }
779 if !word.is_empty() {
780 words.push(word);
781 }
782 for name in words {
783 let output = PathBuf::from(name);
784 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
785 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
786 let artifact_name =
787 output_stem == stem || output_stem == format!("lib{stem}");
788 let artifact_kind = output.extension().is_none_or(|ext| {
789 matches!(
790 ext.to_str(),
791 Some(
792 "rlib"
793 | "rmeta"
794 | "so"
795 | "dylib"
796 | "a"
797 | "lib"
798 | "dll"
799 | "exe"
800 | "pdb"
801 )
802 )
803 });
804 if artifact_name
805 && artifact_kind
806 && output.parent() == Some(dir.as_path())
807 && output.extension().is_none_or(|ext| ext != "d")
808 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
809 && output.canonicalize().is_ok_and(|p| p == output)
810 && artifact_signature(&output)?
811 {
812 outputs.insert(output);
813 }
814 }
815 }
816 }
817 }
818 }
819 Ok(outputs.into_iter().collect())
820}
821
822#[derive(Serialize, Deserialize)]
823struct CargoResidue {
824 target: PathBuf,
825 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
826}
827
828fn residue_files(
831 target: &Path,
832 outputs: &[PathBuf],
833) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
834 let mut files = std::collections::BTreeMap::new();
835 let mut dirs = vec![target.to_owned()];
836 let mut bytes = 0usize;
837 let mut entries = 0usize;
838 while let Some(dir) = dirs.pop() {
839 for entry in std::fs::read_dir(dir)? {
840 let path = entry?.path();
841 entries += 1;
842 ensure!(entries <= 4096, "residue inventory too large");
843 let metadata = std::fs::symlink_metadata(&path)?;
844 ensure!(
845 !metadata.file_type().is_symlink(),
846 "symlink in Cargo residue"
847 );
848 if metadata.is_dir() {
849 dirs.push(path);
850 } else {
851 ensure!(metadata.is_file(), "non-file in Cargo residue");
852 if outputs.contains(&path) {
853 continue;
854 }
855 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
856 let body = std::fs::read(&path)?;
857 bytes = bytes.saturating_add(body.len());
858 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
859 files.insert(path.strip_prefix(target)?.to_owned(), body);
860 }
861 }
862 }
863 Ok(files)
864}
865
866fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
867 Ok(home
868 .join("runs")
869 .join(id)
870 .join("build-residue")
871 .join(worktree.file_name().context("missing worktree name")?))
872}
873
874fn residue_root(target: &Path) -> Result<()> {
877 for entry in std::fs::read_dir(target)? {
878 let name = entry?.file_name();
879 ensure!(
880 matches!(
881 name.to_str(),
882 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
883 ),
884 "unrecognized Cargo residue root entry"
885 );
886 }
887 Ok(())
888}
889
890fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
891 cache_tag(target, false)?;
892 residue_root(target)?;
893 let files = residue_files(target, outputs)?;
894 for (relative, body) in &files {
895 let path = target.join(relative);
896 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
897 let lock = matches!(
898 relative.to_str(),
899 Some("debug/.cargo-lock" | "release/.cargo-lock")
900 ) && body.is_empty();
901 let dep_info = path.extension().is_some_and(|ext| ext == "d")
902 && outputs.iter().any(|output| {
903 output.parent() == path.parent()
904 && (output.file_stem() == path.file_stem()
905 || output.file_stem().is_some_and(|stem| {
906 stem.to_string_lossy()
907 == format!(
908 "lib{}",
909 path.file_stem().unwrap_or_default().to_string_lossy()
910 )
911 }))
912 });
913 ensure!(identity || lock || dep_info, "unproven Cargo residue");
914 }
915 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
916 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
917 let tmp = proof.with_extension("tmp");
918 std::fs::write(
919 &tmp,
920 serde_json::to_vec(&CargoResidue {
921 target: target.to_owned(),
922 files,
923 })?,
924 )?;
925 std::fs::rename(tmp, proof)?;
926 Ok(())
927}
928
929pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
930 let target = cargo_target(worktree).await?;
931 residue_root(&target)?;
932 let locks = cargo_locks(&target)?;
933 let proof: CargoResidue =
934 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
935 ensure!(
936 proof.target == target && proof.files == residue_files(&target, &[])?,
937 "Cargo residue changed or contains unproven files"
938 );
939 Ok(locks)
940}
941
942pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
944 let mut held = Vec::new();
945 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
946 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
947 }) {
948 let lock = target.join(profile).join(name);
949 match std::fs::symlink_metadata(&lock) {
950 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
951 result => {
952 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
953 }
954 }
955 {
956 plain_file(&lock, "Cargo lock")?;
957 let file = OpenOptions::new()
958 .read(true)
959 .write(true)
960 .open(&lock)
961 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
962 crate::pr_head::try_lock_file(&file)
963 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
964 held.push(file);
965 }
966 }
967 Ok(held)
968}
969
970pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
973 let home = home.to_owned();
974 let disk = disk.clone();
975 let runtime = tokio::runtime::Handle::current();
976 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
977 .await
978 .unwrap_or_else(|e| Report {
979 failures: vec![format!("retention worker failed: {e}")],
980 ..Report::default()
981 })
982}
983
984#[derive(Default, Serialize, Deserialize)]
985struct Measurement {
986 at: Option<Timestamp>,
987 bytes: u64,
988}
989
990async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
991 let mut report = Report {
992 checked_at: Some(now),
993 limit_bytes: disk.worktree_limit_bytes,
994 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
995 ..Report::default()
996 };
997 if !cfg!(unix) {
998 report
999 .failures
1000 .push("Automatic output reclamation requires Unix process containment".into());
1001 return report;
1002 }
1003 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
1004 return report;
1005 };
1006 let mut targets = Vec::new();
1007 let mut under_pressure = false;
1008 let mut registrations = std::collections::HashMap::new();
1009 let measurements_path = home.join("output-measurements.json");
1010 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
1011 std::fs::read(&measurements_path)
1012 .ok()
1013 .and_then(|b| serde_json::from_slice(&b).ok())
1014 .unwrap_or_default();
1015 let mut observed = std::collections::BTreeSet::new();
1016 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
1017 .into_iter()
1018 .flatten()
1019 .flatten()
1020 .map(|e| e.file_name().to_string_lossy().into_owned())
1021 .collect();
1022 ids.sort();
1023 for id in ids {
1024 if !crate::run::is_run_id(&id) {
1025 continue;
1026 }
1027 let state = match RunState::load_under(&id, home) {
1028 Ok(state) => state,
1029 Err(e) => {
1030 report
1031 .retained
1032 .push(format!("{id}: unreadable run ownership: {e:#}"));
1033 continue;
1034 }
1035 };
1036 if disk.worktree_limit_bytes == 0 {
1037 continue;
1038 }
1039 match run_paths_missing(&state) {
1040 Ok(true) => continue,
1041 Ok(false) => {}
1042 Err(e) => {
1043 report.failures.push(format!("{id}: {e:#}"));
1044 continue;
1045 }
1046 }
1047 let list = match registrations.entry(state.repo.clone()) {
1048 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1049 std::collections::hash_map::Entry::Vacant(entry) => {
1050 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1051 Ok(list) => entry.insert(list),
1052 Err(e) => {
1053 if matches!(run_paths_missing(&state), Ok(true)) {
1056 continue;
1057 }
1058 report.failures.push(format!("{id}: {e:#}"));
1059 continue;
1060 }
1061 }
1062 }
1063 };
1064 let paths = match registered_worktrees(&state, list) {
1065 Ok(paths) => paths,
1066 Err(e) => {
1067 report.failures.push(format!("{id}: {e:#}"));
1068 continue;
1069 }
1070 };
1071 for path in paths {
1072 if !path.join("target").exists() {
1073 continue;
1074 }
1075 let target = path.join("target");
1076 observed.insert(target.clone());
1077 let pressure =
1078 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1079 under_pressure |= pressure;
1080 let measurement = measurements.entry(target).or_default();
1081 if pressure
1082 || measurement
1083 .at
1084 .is_none_or(|at| crate::clean::due(now, at, 300))
1085 {
1086 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1087 measurement.at = Some(now);
1088 }
1089 let size = measurement.bytes;
1090 report.measured_bytes = report.measured_bytes.saturating_add(size);
1091 targets.push((state.output_activity(), id.clone(), path, size));
1092 }
1093 }
1094 if under_pressure {
1095 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1098 } else {
1099 targets.sort_by_key(|t| t.0);
1100 }
1101 let free_before = crate::disk::free_bytes(home).ok();
1102 #[cfg(unix)]
1103 {
1104 let mut seen = std::collections::BTreeSet::new();
1105 report.measured_allocated_file_bytes = Some(0);
1106 report.estimated_released_allocated_bytes = Some(0);
1107 report.proven_reclaimable_allocated_bytes = Some(0);
1108 for (_, _, path, _) in &targets {
1109 report.measured_allocated_file_bytes = report
1110 .measured_allocated_file_bytes
1111 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1112 .map(|(a, b)| a.saturating_add(b));
1113 }
1114 }
1115 let mut remaining = report.measured_bytes;
1116 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1117 let mut deletion_started = false;
1118 let mut oversized_started = false;
1119 for (updated, id, path, _) in targets {
1120 if disk.worktree_limit_bytes == 0
1121 || remaining <= disk.worktree_limit_bytes
1122 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1123 {
1124 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1125 continue;
1126 }
1127 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1128 report
1129 .retained
1130 .push(format!("{id}: per-pass byte budget spent"));
1131 continue;
1132 }
1133 let result = async {
1134 ensure!(
1135 crate::clean::due(now, updated, disk.fold_grace_secs),
1136 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1137 );
1138 let _owner = crate::pr_head::owner_lock(home, &id)?;
1139 let state = RunState::load_under(&id, home)?;
1140 ensure!(
1141 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1142 == Liveness::Dead,
1143 "run live or ownership unknown"
1144 );
1145 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1146 let _driver = crate::driver::maintenance(home, &id)?;
1147 let _outputs = OutputLease::take(&path, true)?;
1148 ensure!(
1149 worktrees(&state).await?.contains(&path),
1150 "worktree registration changed"
1151 );
1152 let target = validate_cargo_target(&path, true).await
1153 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1154 let _cargo = cargo_locks(&target)
1155 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1156 validate_cargo_target(&path, true).await
1159 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1160 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1161 if lease_meta.len() > 0 {
1162 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1163 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1164 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1165 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1166 }
1167 output_age(&target, now, disk.fold_grace_secs)
1168 .with_context(|| format!("stage output-age: {}", target.display()))?;
1169 let mut outputs = declared_outputs(&target)
1170 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1171 let incremental = match crate::incremental::candidates(&path) {
1172 Ok(candidates) => candidates,
1173 Err(error) => {
1174 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1175 crate::incremental::Reclaim::default()
1176 }
1177 };
1178 let incremental_files = incremental.files.clone();
1179 outputs.extend(incremental_files.iter().cloned());
1180 outputs.sort();
1181 outputs.dedup();
1182 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1183 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1184 .with_context(|| format!("stage measure: {}", p.display())))
1185 .collect::<Result<Vec<_>>>()?;
1186 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1187 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1188 let mut allocation = ReleasedAllocation::of(&outputs)?;
1189 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1190 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1191 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1192 ensure!(!deletion_started || !exceeds_budget,
1196 "declared outputs exceed remaining per-pass byte budget");
1197 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1200 std::fs::write(
1201 &ledger,
1202 serde_json::to_vec(
1203 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1204 )?,
1205 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1206 if !incremental_files.is_empty() {
1209 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1210 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1211 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1212 }
1213 ensure!(worktrees(&state).await?.contains(&path),
1214 "worktree registration changed before deletion");
1215 deletion_started = true;
1216 oversized_started = exceeds_budget;
1217 measurements.remove(&target);
1218 for (output, size) in outputs.iter().zip(sizes) {
1219 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1220 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1221 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1222 report.freed_bytes = report.freed_bytes.saturating_add(size);
1224 remaining = remaining.saturating_sub(size);
1225 }
1226 incremental.finish().context("finish incremental reclamation")?;
1227 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1228 Ok::<_, anyhow::Error>(())
1229 }
1230 .await;
1231 if let Err(e) = result {
1232 report.retained.push(format!(
1233 "{id}/{}: {e:#}",
1234 path.file_name().unwrap_or_default().to_string_lossy()
1235 ));
1236 }
1237 }
1238 measurements.retain(|path, _| observed.contains(path));
1239 if let Ok(body) = serde_json::to_vec(&measurements)
1240 && let Err(e) = std::fs::write(&measurements_path, body)
1241 {
1242 report.failures.push(format!("output measurements: {e}"));
1243 }
1244 copies::evict(home, disk, now, &mut report).await;
1245 retire_empty(home, disk, now, &mut report).await;
1246 report.protected_or_unproven_bytes = report
1247 .measured_bytes
1248 .saturating_sub(report.proven_reclaimable_bytes);
1249 let pressure_remaining = pressure_paths
1250 .iter()
1251 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1252 if !report.failures.is_empty()
1253 || disk.worktree_limit_bytes > 0
1254 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1255 {
1256 let details = report
1257 .retained
1258 .iter()
1259 .filter(|reason| {
1260 !reason.contains("within policy")
1261 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1262 })
1263 .take(4)
1264 .chain(report.failures.iter().take(2))
1265 .cloned()
1266 .collect::<Vec<_>>()
1267 .join("; ");
1268 crate::notices::raise_in(
1269 home,
1270 crate::notices::Notice::warn(
1271 "disk:retention",
1272 format!(
1273 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1274 report.measured_bytes,
1275 report.proven_reclaimable_bytes,
1276 report.protected_or_unproven_bytes,
1277 report.freed_bytes,
1278 if report.proven_reclaimable_bytes == 0 {
1279 "No inspected contents have deletion authority."
1280 } else {
1281 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1282 }
1283 ),
1284 ),
1285 );
1286 }
1287 report.observed_home_free_change_bytes = free_before
1288 .zip(crate::disk::free_bytes(home).ok())
1289 .map(|(before, after)| {
1290 (i128::from(after) - i128::from(before))
1291 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1292 });
1293 let store = home.join("retention.json");
1294 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1295 let tmp = home.join("retention.json.tmp");
1296 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1297 tracing::warn!("retention report could not be saved: {e}");
1298 }
1299 }
1300 tracing::info!(
1301 measured_bytes = report.measured_bytes,
1302 freed_bytes = report.freed_bytes,
1303 retired = report.retired,
1304 retained = report.retained.len(),
1305 failures = report.failures.len(),
1306 "retained worktree cleanup"
1307 );
1308 for reason in report.retained.iter().chain(&report.failures) {
1309 tracing::debug!(%reason, "worktree retained");
1310 }
1311 report
1312}
1313
1314async fn retire_empty(
1315 home: &Path,
1316 disk: &crate::config::Disk,
1317 now: Timestamp,
1318 report: &mut Report,
1319) {
1320 if !disk.auto_fold {
1321 return;
1322 }
1323 let queue = crate::queue::Queue::at(home.join("queue"));
1324 let questions = crate::ask::Questions::at(home.join("questions"));
1325 let open = questions.list();
1326 for listed in queue.list() {
1327 let Ok(_claim) = queue.claim(&listed.id) else {
1328 continue;
1329 };
1330 let Ok(task) = queue.get(&listed.id) else {
1331 continue;
1332 };
1333 if open
1334 .iter()
1335 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1336 {
1337 report.retained.push(format!(
1338 "{}: task has an unanswered owner decision",
1339 task.id
1340 ));
1341 continue;
1342 }
1343 let sources: Vec<_> = task
1344 .runs
1345 .iter()
1346 .collect::<std::collections::BTreeSet<_>>()
1347 .into_iter()
1348 .cloned()
1349 .collect();
1350 let mut useful = Vec::new();
1351 let mut selected = Vec::new();
1352 let mut public = Vec::new();
1353 let mut unreadable = false;
1354 for id in &sources {
1355 let Ok(state) = RunState::load_under(id, home) else {
1356 unreadable = true;
1357 continue;
1358 };
1359 if state.released() || state.retired_to.is_some() {
1360 continue;
1361 }
1362 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1363 selected.push(state.clone());
1364 }
1365 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1366 public.push(state.clone());
1367 }
1368 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1369 useful.push(state);
1370 }
1371 }
1372 let pool = if !selected.is_empty() {
1376 selected
1377 } else if !public.is_empty() {
1378 public
1379 } else {
1380 useful
1381 };
1382 if unreadable || pool.len() != 1 {
1383 report.retained.push(format!(
1384 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1385 task.id,
1386 sources.len()
1387 ));
1388 continue;
1389 }
1390 let authority = &pool[0];
1391 if authority.candidates.is_empty() {
1392 continue;
1393 }
1394 let latest = &authority.id;
1395 for id in &sources {
1396 if id == latest || report.retired >= 5 {
1397 continue;
1398 }
1399 if let Ok(state) = RunState::load_under(id, home)
1400 && state.retired_to.as_ref() == Some(latest)
1401 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1402 {
1403 continue;
1404 }
1405 let result = async {
1406 ensure!(
1407 !task
1408 .resume_override
1409 .as_ref()
1410 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1411 "operator-pinned recovery run"
1412 );
1413 ensure!(
1414 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1415 "selected replacement recovery source"
1416 );
1417 let _owner = crate::pr_head::owner_lock(home, id)?;
1418 let mut state = RunState::load_under(id, home)?;
1419 ensure!(
1420 state.repo == authority.repo,
1421 "repository differs from recovery authority"
1422 );
1423 ensure!(
1424 !crate::pr_recovery::explicit_sources_retained(home, id),
1425 "explicitly retained recovery source"
1426 );
1427 ensure!(
1428 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1429 == Liveness::Dead,
1430 "run live or ownership unknown"
1431 );
1432 let _driver = crate::driver::maintenance(home, id)?;
1433 ensure!(
1434 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1435 "within grace period"
1436 );
1437 ensure!(
1438 !open.iter().any(|q| q.run == *id && q.status.open()),
1439 "unanswered owner decision"
1440 );
1441 ensure!(
1442 state.pr.is_none()
1443 && state.pr_head.is_none()
1444 && state.pr_recovery.is_none()
1445 && state.land_approval.is_none()
1446 && state.merge_decision.is_none()
1447 && state.reviews.is_empty()
1448 && state.recovery.is_none()
1449 && state.candidate_preservation.is_empty()
1450 && state.interrupted_implementations.is_empty()
1451 && !state.released(),
1452 "PR ownership or recovery checkpoint"
1453 );
1454 let proof = prove_empty(home, &state, latest).await?;
1455 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1456 for path in &proof.paths {
1459 ensure!(
1460 git::worktree_remove_clean(&state.repo, path).await?,
1461 "Git refused clean worktree removal"
1462 );
1463 }
1464 Ok::<_, anyhow::Error>(())
1465 }
1466 .await;
1467 match result {
1468 Ok(()) => report.retired += 1,
1469 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1470 }
1471 }
1472 }
1473}
1474
1475struct EmptyProof {
1476 paths: Vec<PathBuf>,
1477 _outputs: Vec<OutputLease>,
1478 _cargo: Vec<File>,
1479}
1480
1481pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1484 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1485 let mut files = Vec::new();
1486 let mut blobs = Vec::new();
1487 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1488 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1489 let parts: Vec<_> = header.split_whitespace().collect();
1490 ensure!(
1491 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1492 "unsupported tracked source kind"
1493 );
1494 let path = worktree.join(name);
1495 let meta = std::fs::symlink_metadata(&path)?;
1496 ensure!(
1497 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1498 "noncanonical tracked source"
1499 );
1500 #[cfg(unix)]
1501 {
1502 use std::os::unix::fs::PermissionsExt as _;
1503 ensure!(
1504 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1505 "tracked executable mode differs from original base"
1506 );
1507 }
1508 path.to_str().context("non UTF-8 source")?;
1509 files.push(git::path_arg(&path));
1510 blobs.push(parts[2].to_owned());
1511 }
1512 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1513 let mut args = vec!["hash-object", "--no-filters", "--"];
1514 args.extend(paths.iter().map(String::as_str));
1515 let actual = git::git(worktree, &args).await?;
1516 ensure!(
1517 actual.lines().eq(expected.iter().map(String::as_str)),
1518 "tracked bytes differ from original base"
1519 );
1520 }
1521 Ok(())
1522}
1523
1524async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1525 let id = &state.id;
1526 let paths = worktrees(state).await?;
1527 let mut guards = Vec::new();
1528 let mut cargo_guards = Vec::new();
1529 for path in &paths {
1530 guards.push(OutputLease::take(path, true)?);
1531 ensure!(
1532 git::rev_parse(path, "HEAD").await? == state.base_commit,
1533 "unique or unrecorded commits"
1534 );
1535 ensure!(
1536 git::git(
1537 path,
1538 &[
1539 "--no-optional-locks",
1540 "status",
1541 "--porcelain",
1542 "--untracked-files=all"
1543 ]
1544 )
1545 .await?
1546 .is_empty(),
1547 "tracked or untracked source changes"
1548 );
1549 ensure!(
1550 git::git(
1551 path,
1552 &[
1553 "--no-optional-locks",
1554 "status",
1555 "--porcelain",
1556 "--untracked-files=all",
1557 "--ignored",
1558 "--",
1559 ".",
1560 ":(exclude)target"
1561 ]
1562 )
1563 .await?
1564 .is_empty(),
1565 "source changes, ignored files or build outputs"
1566 );
1567 tracked_base(path, &state.base_commit).await?;
1568 match std::fs::symlink_metadata(path.join("target")) {
1569 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1570 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1571 Err(error) => return Err(error.into()),
1572 }
1573 for checkpoint in [
1574 "rebase-merge",
1575 "rebase-apply",
1576 "sequencer",
1577 "BISECT_LOG",
1578 "REVERT_HEAD",
1579 "AUTO_MERGE",
1580 "MERGE_MSG",
1581 "CHERRY_PICK_HEAD",
1582 "MERGE_HEAD",
1583 "REBASE_HEAD",
1584 ] {
1585 let location = git::git(
1586 path,
1587 &[
1588 "rev-parse",
1589 "--path-format=absolute",
1590 "--git-path",
1591 checkpoint,
1592 ],
1593 )
1594 .await?;
1595 ensure!(
1596 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1597 "retained Git recovery metadata: {checkpoint}"
1598 );
1599 }
1600 let origin_path = git::git(
1603 path,
1604 &[
1605 "rev-parse",
1606 "--path-format=absolute",
1607 "--git-path",
1608 "ORIG_HEAD",
1609 ],
1610 )
1611 .await?;
1612 match std::fs::symlink_metadata(origin_path.trim()) {
1613 Ok(meta) => {
1614 ensure!(
1615 meta.is_file() && !meta.file_type().is_symlink(),
1616 "unknown ORIG_HEAD artifact"
1617 );
1618 git::rev_parse(path, "ORIG_HEAD")
1619 .await
1620 .context("unreadable ORIG_HEAD")?;
1621 }
1622 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1623 Err(e) => return Err(e.into()),
1624 }
1625 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1626 ensure!(
1627 git::git_raw(
1628 path,
1629 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1630 )
1631 .await?
1632 .ok(),
1633 "ORIG_HEAD retains commits outside the original base"
1634 );
1635 }
1636 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1637 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1638 ensure!(
1639 git::git_raw(
1640 path,
1641 &["merge-base", "--is-ancestor", head, &state.base_commit]
1642 )
1643 .await?
1644 .ok(),
1645 "worktree reflog retains commits outside the original base"
1646 );
1647 }
1648 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1649 ensure!(
1650 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1651 .await?
1652 .ok(),
1653 "unfinished Git operation"
1654 );
1655 }
1656 }
1657 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1660 return Ok(EmptyProof {
1661 paths,
1662 _outputs: guards,
1663 _cargo: cargo_guards,
1664 });
1665 }
1666 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1667 for candidate in &state.candidates {
1668 ensure!(
1669 (paths.contains(&candidate.worktree)
1670 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1671 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1672 "candidate recovery source"
1673 );
1674 }
1675 let pattern = format!("refs/heads/magi/{}/", state.short());
1676 let branches = git::git(
1677 &state.repo,
1678 &["for-each-ref", "--format=%(objectname)", &pattern],
1679 )
1680 .await?;
1681 ensure!(
1682 branches.lines().all(|head| head == state.base_commit),
1683 "unrecorded candidate branch changes"
1684 );
1685 Ok(EmptyProof {
1686 paths,
1687 _outputs: guards,
1688 _cargo: cargo_guards,
1689 })
1690}
1691
1692fn commit_retirement(
1697 home: &Path,
1698 queue: &crate::queue::Queue,
1699 expected: &crate::queue::Task,
1700 state: &mut RunState,
1701 authority: &str,
1702 sources: &[String],
1703 now: Timestamp,
1704) -> Result<()> {
1705 let _write = queue.lock_task(&expected.id)?;
1706 let current = queue.get(&expected.id)?;
1707 ensure!(
1708 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1709 "task changed during retirement proof; retaining worktrees for the current owner decision"
1710 );
1711 ensure!(
1712 !current
1713 .resume_override
1714 .as_ref()
1715 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1716 && !current
1717 .replacement
1718 .as_ref()
1719 .is_some_and(|pin| pin.run == state.id),
1720 "current owner selected this recovery source"
1721 );
1722 ensure!(
1723 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1724 "explicitly retained recovery source"
1725 );
1726 ensure!(
1727 !crate::ask::Questions::at(home.join("questions"))
1728 .list()
1729 .iter()
1730 .any(|q| q.status.open()
1731 && (q.run == state.id
1732 || q.run == current.id
1733 || current.blocked_by.contains(&q.id))),
1734 "owner decision arrived during retirement proof"
1735 );
1736 let dir = home.join("recovery-lineages");
1737 std::fs::create_dir_all(&dir)?;
1738 let tmp = dir.join(format!("{}.tmp", current.id));
1739 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1740 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1741 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1742 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1743 state.retired_to = Some(authority.to_owned());
1744 state.event(
1745 "retention",
1746 format!(
1747 "empty attempt retired to {authority}; recovery sources: {}",
1748 sources.join(", ")
1749 ),
1750 );
1751 state.save_under(home)?;
1752 Ok(())
1754}
1755
1756pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1760 if min_free == 0 {
1761 return std::future::pending().await;
1762 }
1763 loop {
1764 for path in &paths {
1765 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1766 match crate::disk::free_bytes(existing) {
1767 Ok(free) if free >= min_free / 2 => {}
1768 Ok(free) => {
1769 return format!(
1770 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1771 );
1772 }
1773 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1774 }
1775 }
1776 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1777 }
1778}
1779
1780#[cfg(all(test, unix))]
1781mod tests {
1782 use super::*;
1783 use crate::{
1784 config::{Config, Disk},
1785 queue::{Source, Task},
1786 run::{Candidate, RunStatus},
1787 };
1788
1789 struct Fx {
1790 _temp: tempfile::TempDir,
1791 home: PathBuf,
1792 repo: PathBuf,
1793 bay: PathBuf,
1794 base: String,
1795 }
1796 impl Fx {
1797 async fn new() -> Self {
1798 let temp = tempfile::tempdir().unwrap();
1799 let root = temp.path().canonicalize().unwrap();
1800 let home = root.join("home");
1801 let repo = root.join("repo");
1802 let bay = root.join("worktrees");
1803 std::fs::create_dir_all(&repo).unwrap();
1804 std::fs::create_dir_all(home.join("runs")).unwrap();
1805 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1806 git::git(&repo, &["config", "user.name", "Fixture"])
1807 .await
1808 .unwrap();
1809 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1810 .await
1811 .unwrap();
1812 git::git(&repo, &["config", "commit.gpgsign", "false"])
1813 .await
1814 .unwrap();
1815 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1816 std::fs::write(repo.join("source"), "base\n").unwrap();
1817 git::git(&repo, &["add", "."]).await.unwrap();
1818 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1819 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1820 Self {
1821 _temp: temp,
1822 home,
1823 repo,
1824 bay,
1825 base,
1826 }
1827 }
1828 async fn run(&self, tag: &str, useful: bool) -> RunState {
1829 let mut cfg = Config::default();
1830 cfg.graph.worktree_root = Some(self.bay.clone());
1831 let mut state = RunState::new(
1832 self.repo.clone(),
1833 "main".into(),
1834 self.base.clone(),
1835 "task".into(),
1836 cfg,
1837 );
1838 state.id = format!("20260101-000000-{tag}");
1839 state.status = RunStatus::Blocked;
1840 state.driver_exited = true;
1841 let wt = state.worktree_root().join("cand-A");
1842 let branch = format!("magi/{tag}/A");
1843 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1844 .await
1845 .unwrap();
1846 if useful {
1847 std::fs::write(wt.join("source"), tag).unwrap();
1848 git::git(&wt, &["add", "."]).await.unwrap();
1849 git::git(&wt, &["commit", "-m", "candidate change"])
1850 .await
1851 .unwrap();
1852 }
1853 state.candidates.push(Candidate {
1854 index: 0,
1855 label: 'A',
1856 agent: "fixture".into(),
1857 branch,
1858 worktree: wt,
1859 summary: String::new(),
1860 stat: String::new(),
1861 files: usize::from(useful),
1862 commits: usize::from(useful),
1863 empty: !useful,
1864 failed: None,
1865 verified_noop: None,
1866 duration_ms: 0,
1867 folded: false,
1868 });
1869 state.save_under(&self.home).unwrap();
1870 state
1871 }
1872 fn disk() -> Disk {
1873 Disk {
1874 worktree_limit_bytes: 1,
1875 worktree_reclaim_bytes: 1024 * 1024,
1876 fold_grace_secs: 0,
1877 min_free_bytes: 0,
1878 ..Disk::default()
1879 }
1880 }
1881 fn target(wt: &Path) {
1882 let target = wt.join("target");
1883 std::fs::create_dir_all(target.join("debug")).unwrap();
1884 let mut artifact = [1; 200];
1885 artifact[..4].copy_from_slice(b"\x7fELF");
1886 std::fs::write(target.join("debug/output"), artifact).unwrap();
1887 let output = target
1888 .join("debug/output")
1889 .display()
1890 .to_string()
1891 .replace('\\', "\\\\")
1892 .replace(' ', "\\ ");
1893 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1894 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1895 std::fs::write(
1896 target.join("CACHEDIR.TAG"),
1897 "Signature: 8a477f597d28d172789f06886806bc55\n",
1898 )
1899 .unwrap();
1900 std::fs::write(
1901 target.join(".rustc_info.json"),
1902 "{\"rustc_fingerprint\":123}",
1903 )
1904 .unwrap();
1905 }
1906 fn now() -> Timestamp {
1907 Timestamp::now() + jiff::SignedDuration::from_secs(2)
1908 }
1909 fn task(&self, runs: &[RunState]) -> Task {
1910 let mut task = Task::new(
1911 "fixture".into(),
1912 "task".into(),
1913 self.repo.clone(),
1914 Source::Human,
1915 );
1916 task.runs = runs.iter().map(|s| s.id.clone()).collect();
1917 crate::queue::Queue::at(self.home.join("queue"))
1918 .put(&mut task)
1919 .unwrap();
1920 task
1921 }
1922 }
1923
1924 #[cfg(unix)]
1925 #[tokio::test]
1926 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1927 let f = Fx::new().await;
1928 let state = f.run("ca01", true).await;
1929 let wt = &state.candidates[0].worktree;
1930 let generation = crate::incremental::tests::generate(wt);
1931 for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1933 for entry in std::fs::read_dir(dir).unwrap() {
1934 let path = entry.unwrap().path();
1935 if path.extension().is_some_and(|x| x == "d") {
1936 std::fs::remove_file(path).unwrap();
1937 }
1938 }
1939 }
1940 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1941 assert_eq!(
1942 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1943 0,
1944 "live generation is protected"
1945 );
1946 drop(generation);
1947 let candidates = crate::incremental::candidates(wt).unwrap();
1948 let logical: u64 = candidates
1949 .files
1950 .iter()
1951 .map(|p| std::fs::metadata(p).unwrap().len())
1952 .sum();
1953 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1954 std::fs::write(&unknown, "retain unknown data").unwrap();
1955 drop(candidates);
1956 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1957 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1958 assert!(
1959 refused
1960 .retained
1961 .iter()
1962 .any(|s| s.contains("provenance mismatch"))
1963 );
1964 std::fs::remove_file(&unknown).unwrap();
1965 let legacy = wt.join("target/debug/incremental/legacy-session");
1966 std::fs::create_dir(&legacy).unwrap();
1967 let candidates = crate::incremental::candidates(wt).unwrap();
1969 let bin = candidates
1970 .files
1971 .iter()
1972 .find(|p| p.extension().is_some_and(|x| x == "bin"))
1973 .unwrap();
1974 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1975 drop(candidates);
1976 let cargo = cargo_locks(&wt.join("target")).unwrap();
1977 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1978 drop(cargo);
1979 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1980 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1981 drop(outputs);
1982 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1983 assert_eq!(report.freed_bytes, logical, "{report:?}");
1984 assert_eq!(report.proven_reclaimable_bytes, logical);
1985 assert!(report.measured_bytes > logical);
1986 assert_eq!(
1987 report.protected_or_unproven_bytes,
1988 report.measured_bytes - logical
1989 );
1990 #[cfg(unix)]
1991 {
1992 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
1993 assert!(
1994 report.measured_allocated_file_bytes.unwrap()
1995 >= report.estimated_released_allocated_bytes.unwrap()
1996 );
1997 }
1998 assert!(report.observed_home_free_change_bytes.is_some());
1999 assert!(legacy.join("query-cache.bin").is_file());
2000 assert!(wt.join("src/lib.rs").is_file());
2001 assert_eq!(report.retired, 0);
2002 }
2003
2004 #[cfg(unix)]
2005 #[tokio::test]
2006 async fn incremental_and_deps_links_account_for_retained_third_link() {
2007 use std::os::unix::fs::MetadataExt;
2008 let f = Fx::new().await;
2009 let state = f.run("ca02", true).await;
2010 let wt = &state.candidates[0].worktree;
2011 let generation = crate::incremental::tests::generate(wt);
2012 drop(generation);
2013 let candidates = crate::incremental::candidates(wt).unwrap();
2014 let objects = candidates
2015 .files
2016 .iter()
2017 .filter(|p| p.extension().is_some_and(|x| x == "o"))
2018 .take(2)
2019 .cloned()
2020 .collect::<Vec<_>>();
2021 assert_eq!(
2022 objects.len(),
2023 2,
2024 "fixture must produce real incremental objects"
2025 );
2026 let mut outputs = candidates.files.clone();
2027 drop(candidates);
2028 let target = wt.join("target");
2029 for dir in [target.join("debug"), target.join("debug/deps")] {
2030 for entry in std::fs::read_dir(dir).unwrap() {
2031 let path = entry.unwrap().path();
2032 if path.extension().is_some_and(|x| x == "d") {
2033 std::fs::remove_file(path).unwrap();
2034 }
2035 }
2036 }
2037 for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
2040 let path = entry.unwrap().path();
2041 if path.is_file() {
2042 std::fs::remove_file(path).unwrap();
2043 }
2044 }
2045 for (index, object) in objects.iter().enumerate() {
2046 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2047 let dep = target.join(format!("debug/deps/link{index}"));
2048 std::fs::hard_link(object, &dep).unwrap();
2049 std::fs::write(
2050 dep.with_extension("d"),
2051 format!("{}: src/lib.rs\n", dep.display()),
2052 )
2053 .unwrap();
2054 outputs.push(dep);
2055 }
2056 let retained = wt.join("retained-input");
2057 std::fs::hard_link(&objects[1], &retained).unwrap();
2058 let sentinel = std::fs::read(&retained).unwrap();
2059 let logical: u64 = outputs
2060 .iter()
2061 .map(|p| std::fs::metadata(p).unwrap().len())
2062 .sum();
2063 let mut seen = std::collections::BTreeSet::new();
2064 let retained_meta = std::fs::metadata(&retained).unwrap();
2065 let retained_inode = (retained_meta.dev(), retained_meta.ino());
2066 let mut expected = 0u64;
2067 for output in &outputs {
2068 let m = std::fs::metadata(output).unwrap();
2069 let inode = (m.dev(), m.ino());
2070 if inode != retained_inode && seen.insert(inode) {
2071 expected += m.blocks() * 512;
2072 }
2073 }
2074 assert!(expected > 0);
2075 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2076 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2077 assert_eq!(report.freed_bytes, logical, "{report:?}");
2078 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2079 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2080 assert!(outputs.iter().all(|p| !p.exists()));
2081 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2082 assert!(wt.join("src/lib.rs").is_file());
2083 }
2084
2085 #[tokio::test]
2086 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2087 let f = Fx::new().await;
2088 let state = f.run("ca03", true).await;
2089 let wt = &state.candidates[0].worktree;
2090 Fx::target(wt);
2091 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2092 let unknown = wt.join("target/debug/operator-note");
2093 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2094 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2095 assert_eq!(report.proven_reclaimable_bytes, 0);
2096 #[cfg(unix)]
2097 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2098 assert!(report.protected_or_unproven_bytes >= 4096);
2099 let message = || {
2100 crate::notices::Notices::at(f.home.join("notifications"))
2101 .list()
2102 .into_iter()
2103 .find(|n| n.key == "disk:retention")
2104 .unwrap()
2105 .message
2106 };
2107 let notice = message();
2108 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2109 assert!(notice.contains("protected or unproven"), "{notice}");
2110 assert!(
2111 notice.contains("legacy incremental/build and unknown files retained"),
2112 "{notice}"
2113 );
2114 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2115 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2116 assert_eq!(report.freed_bytes, 0);
2117 let notice = message();
2118 assert!(notice.contains(&state.id), "{notice}");
2119 assert!(
2120 notice.contains("lock") || notice.contains("owner"),
2121 "{notice}"
2122 );
2123 drop(owner);
2124 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2125 }
2126
2127 #[tokio::test]
2128 async fn real_cargo_build_holds_the_target_against_reclamation() {
2129 let f = Fx::new().await;
2130 let state = f.run("ca02", true).await;
2131 let wt = &state.candidates[0].worktree;
2132 Fx::target(wt);
2133 std::fs::create_dir(wt.join("src")).unwrap();
2134 std::fs::write(
2135 wt.join("Cargo.toml"),
2136 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2137 )
2138 .unwrap();
2139 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2140 std::fs::write(
2141 wt.join("build.rs"),
2142 r#"fn main() {
2143 println!("cargo:rerun-if-env-changed=FIXTURE_READY");
2144 let Ok(ready) = std::env::var("FIXTURE_READY") else { return };
2145 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2146 std::fs::write(ready, "ready").unwrap();
2147 let start = std::time::Instant::now();
2148 while !std::path::Path::new(&release).exists() {
2149 assert!(start.elapsed().as_secs() < 120, "fixture was not released");
2150 std::thread::sleep(std::time::Duration::from_millis(10));
2151 }
2152 }"#,
2153 )
2154 .unwrap();
2155 let ready = wt.join("ready");
2156 let release = wt.join("release");
2157 let mut cargo = tokio::process::Command::new("cargo");
2158 cargo.env_clear();
2159 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2160 if let Some(value) = std::env::var_os(key) {
2161 cargo.env(key, value);
2162 }
2163 }
2164 cargo
2165 .current_dir(wt)
2166 .env("RUSTUP_TOOLCHAIN", "stable")
2167 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2168 .env("CARGO_INCREMENTAL", "1")
2169 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2170 .arg(wt.join("target"));
2171 let warm = cargo.output().await.unwrap();
2175 assert!(
2176 warm.status.success(),
2177 "fixture compilation failed: {}",
2178 String::from_utf8_lossy(&warm.stderr)
2179 );
2180 let log_path = f._temp.path().join("live-cargo.stderr");
2181 let mut child = cargo
2182 .env("FIXTURE_READY", &ready)
2183 .env("FIXTURE_RELEASE", &release)
2184 .stdout(std::process::Stdio::null())
2185 .stderr(std::fs::File::create(&log_path).unwrap())
2186 .kill_on_drop(true)
2187 .spawn()
2188 .unwrap();
2189 let readiness = tokio::time::timeout(std::time::Duration::from_secs(60), async {
2190 while !ready.exists() {
2191 if let Some(status) = child.try_wait().unwrap() {
2192 return Err(format!("fixture exited before readiness: {status}"));
2193 }
2194 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2195 }
2196 Ok(())
2197 })
2198 .await;
2199 let arrived = matches!(readiness, Ok(Ok(())));
2200 let report = if arrived {
2201 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2202 } else {
2203 None
2204 };
2205 std::fs::write(release, "release").unwrap();
2207 let status = tokio::time::timeout(std::time::Duration::from_secs(60), child.wait()).await;
2208 if status.is_err() {
2209 child.kill().await.unwrap();
2210 child.wait().await.unwrap();
2211 }
2212 assert!(
2213 arrived,
2214 "fixture build never reached its active build script: {readiness:?}\n{}",
2215 std::fs::read_to_string(&log_path).unwrap()
2216 );
2217 assert!(
2218 status.unwrap().unwrap().success(),
2219 "fixture build failed: {}",
2220 std::fs::read_to_string(&log_path).unwrap()
2221 );
2222 let report = report.unwrap();
2223 assert_eq!(report.freed_bytes, 0, "{report:?}");
2224 assert!(
2225 report
2226 .retained
2227 .iter()
2228 .any(|s| s.contains("Cargo build is active")),
2229 "{report:?}"
2230 );
2231 assert!(wt.join("target/debug/output").is_file());
2232 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2233 }
2234
2235 #[cfg(unix)]
2236 #[test]
2237 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2238 let temp = tempfile::tempdir().unwrap();
2239 let a = temp.path().join("a");
2240 let b = temp.path().join("b");
2241 let outside = temp.path().join("retained-input");
2242 std::fs::write(&a, vec![1; 65536]).unwrap();
2243 std::fs::hard_link(&a, &b).unwrap();
2244 std::fs::hard_link(&a, &outside).unwrap();
2245 let mut seen = Default::default();
2246 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2247 assert!(blocks > 0);
2248 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2249 std::fs::remove_file(&a).unwrap();
2250 assert_eq!(accounting.unlinked(&a), Some(0));
2251 std::fs::remove_file(&b).unwrap();
2252 assert_eq!(accounting.unlinked(&b), Some(0));
2253 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2254 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2255 std::fs::remove_file(&outside).unwrap();
2256 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2257 }
2258
2259 #[cfg(unix)]
2260 #[test]
2261 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2262 let temp = tempfile::tempdir_in(".").unwrap();
2263 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2264 assert!(home.is_relative());
2265 let paths = vec![std::env::current_dir().unwrap()];
2266 let gib = 1024 * 1024 * 1024;
2267 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2268 assert!(
2269 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2270 9 * gib
2271 ))
2272 .is_err()
2273 );
2274 drop(reservation);
2275 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2276 let bad = temp.path().join("symlink-state");
2277 std::fs::create_dir(&bad).unwrap();
2278 std::os::unix::fs::symlink(
2279 home.canonicalize().unwrap().join("disk-admission"),
2280 bad.join("disk-admission"),
2281 )
2282 .unwrap();
2283 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2284 }
2285
2286 #[cfg(unix)]
2287 #[test]
2288 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2289 let temp = tempfile::tempdir().unwrap();
2290 let home = temp.path().canonicalize().unwrap();
2291 let paths = vec![home.clone(), home.join("not-created-yet")];
2292 let gib = 1024 * 1024 * 1024;
2293 let floor = 8 * gib;
2294 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2295 assert_eq!(first.len(), 1);
2296 let blocked =
2297 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2298 assert!(blocked.to_string().contains("active writer reservations"));
2299 let second =
2300 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2301 drop(first);
2302 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2303 drop(second);
2304 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2305 assert!(
2306 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2307 );
2308 }
2309
2310 #[cfg(unix)]
2311 #[test]
2312 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2313 let temp = tempfile::tempdir().unwrap();
2314 let home = temp.path().canonicalize().unwrap();
2315 let barrier = std::sync::Barrier::new(2);
2316 let gib = 1024 * 1024 * 1024;
2317 let results = std::thread::scope(|scope| {
2318 let threads: Vec<_> = (0..2)
2319 .map(|_| {
2320 scope.spawn(|| {
2321 barrier.wait();
2322 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2323 Ok(100 + gib)
2324 })
2325 })
2326 })
2327 .collect();
2328 threads
2329 .into_iter()
2330 .map(|thread| thread.join().unwrap())
2331 .collect::<Vec<_>>()
2332 });
2333 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2334 assert!(
2335 results
2336 .iter()
2337 .filter_map(|r| r.as_ref().err())
2338 .all(|e| e.to_string().contains("active writer reservations"))
2339 );
2340 }
2341
2342 #[tokio::test]
2343 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2344 real_cargo_fixture(false).await;
2345 }
2346
2347 #[tokio::test]
2348 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2349 real_cargo_fixture(true).await;
2350 }
2351
2352 async fn real_cargo_fixture(precreate: bool) {
2353 let f = Fx::new().await;
2354 let state = f.run("ca00", true).await;
2355 let wt = &state.candidates[0].worktree;
2356 let source = "fn main() { println!(\"retained fixture\"); }\n";
2357 std::fs::create_dir(wt.join("src")).unwrap();
2358 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2359 std::fs::write(
2360 wt.join("Cargo.toml"),
2361 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2362 )
2363 .unwrap();
2364 if precreate {
2365 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2366 }
2367 let mut cargo = std::process::Command::new("cargo");
2368 cargo.env_clear();
2369 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2370 if let Some(value) = std::env::var_os(key) {
2371 cargo.env(key, value);
2372 }
2373 }
2374 let build = cargo
2375 .current_dir(wt)
2376 .env("RUSTUP_TOOLCHAIN", "stable")
2377 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2378 .env("CARGO_INCREMENTAL", "0")
2379 .args([
2380 "build",
2381 "--offline",
2382 "--release",
2383 "--jobs",
2384 "1",
2385 "--target-dir",
2386 ])
2387 .arg(wt.join("target"))
2388 .output()
2389 .unwrap();
2390 assert!(
2391 build.status.success(),
2392 "{}",
2393 String::from_utf8_lossy(&build.stderr)
2394 );
2395 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2396 if precreate {
2397 let again = cargo.output().unwrap();
2399 assert!(
2400 again.status.success(),
2401 "{}",
2402 String::from_utf8_lossy(&again.stderr)
2403 );
2404 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2405 assert!(cargo_target(wt).await.is_err());
2406 }
2407 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2408 .await
2409 .unwrap();
2410 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2411 .await
2412 .unwrap();
2413 let artifact = wt.join("target/release/retained-fixture");
2414 assert!(artifact.is_file());
2415 let unknown = wt.join("target/release/operator-note");
2416 std::fs::write(&unknown, "keep this note\n").unwrap();
2417
2418 let target = wt.join("target");
2419 let mut preserved = std::collections::BTreeMap::new();
2420 for relative in [
2421 "tmp/sentinel",
2422 "magi-tools/sentinel",
2423 "lifecycle-client-check.log",
2424 ] {
2425 let path = target.join(relative);
2426 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2427 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2428 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2429 }
2430 let decoy = target.join("magi-tools/retained-fixture");
2433 std::fs::copy(&artifact, &decoy).unwrap();
2434 let dep_info = decoy.with_extension("d");
2435 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2436 for path in [
2437 decoy,
2438 dep_info,
2439 wt.join("Cargo.toml"),
2440 wt.join("Cargo.lock"),
2441 ] {
2442 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2443 }
2444 let proven = declared_outputs(&target).unwrap();
2445
2446 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2447 assert!(out.freed_bytes > 0, "{out:?}");
2448 for (path, bytes) in preserved {
2449 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2450 }
2451 assert!(!artifact.exists(), "{out:?}");
2452 assert_eq!(
2453 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2454 source
2455 );
2456 assert_eq!(
2457 std::fs::read_to_string(&unknown).unwrap(),
2458 "keep this note\n"
2459 );
2460 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2461 let ledger: serde_json::Value = serde_json::from_slice(
2462 &std::fs::read(
2463 f.home
2464 .join("runs")
2465 .join(&state.id)
2466 .join("build-outputs.json"),
2467 )
2468 .unwrap(),
2469 )
2470 .unwrap();
2471 assert_eq!(
2472 ledger["identity"],
2473 if precreate {
2474 "rustc-profile-dep-info"
2475 } else {
2476 "cache-tag"
2477 }
2478 );
2479 if precreate {
2480 assert!(!target.join("CACHEDIR.TAG").exists());
2481 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2482 }
2483 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2484 assert!(deleted.contains(&artifact));
2485 assert_eq!(deleted, proven);
2486 for path in deleted {
2487 assert!(!path.exists());
2488 assert!(
2489 [target.join("release"), target.join("release/deps")]
2490 .contains(&path.parent().unwrap().to_owned())
2491 );
2492 }
2493 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2494 }
2495
2496 fn missing_tag_target(wt: &Path) {
2497 Fx::target(wt);
2498 let target = wt.join("target");
2499 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2500 for subdir in ["debug/deps", "debug/.fingerprint"] {
2501 std::fs::create_dir_all(target.join(subdir)).unwrap();
2502 }
2503 std::fs::write(target.join(".rustc_info.json"),
2504 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2505 }
2506
2507 #[tokio::test]
2508 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2509 for legacy in [false, true] {
2510 let f = Fx::new().await;
2511 let mut state = f.run("ac00", true).await;
2512 Fx::target(&state.candidates[0].worktree);
2513 let original = state.updated_at;
2514 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2515 if legacy {
2516 let path = f.home.join("runs").join(&state.id).join("run.json");
2517 let mut value = serde_json::to_value(&state).unwrap();
2518 value.as_object_mut().unwrap().remove("output_activity_at");
2519 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2520 state = RunState::load_under(&state.id, &f.home).unwrap();
2521 assert_eq!(state.output_activity(), original);
2522 }
2523 let age = state.output_activity();
2524 for offset in [30, 20, 10] {
2525 state
2526 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2527 .unwrap();
2528 state = RunState::load_under(&state.id, &f.home).unwrap();
2529 assert_eq!(state.output_activity(), age);
2530 }
2531 let mut disk = Fx::disk();
2532 disk.fold_grace_secs = 3600;
2533 let report = sweep(&f.home, &disk, now).await;
2534 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2535 }
2536 }
2537
2538 #[tokio::test]
2539 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2540 for evidence in [
2541 "execution",
2542 "artifact",
2543 "dep-info",
2544 "fingerprint",
2545 "lease",
2546 "future",
2547 ] {
2548 let f = Fx::new().await;
2549 let mut state = f.run("ac01", true).await;
2550 let wt = state.candidates[0].worktree.clone();
2551 missing_tag_target(&wt);
2552 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2553 let recent = now - jiff::SignedDuration::from_secs(10);
2554 match evidence {
2555 "execution" => {
2556 state.note_output_activity(recent);
2557 state.save_under_at(&f.home, recent).unwrap();
2558 }
2559 "lease" => {
2560 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2561 let file = lease._file.try_clone().unwrap();
2562 drop(lease);
2563 file.set_modified(std::time::SystemTime::from(recent))
2564 .unwrap();
2565 }
2566 _ => {
2567 let relative = match evidence {
2568 "dep-info" => "debug/output.d",
2569 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2570 _ => "debug/output",
2571 };
2572 let path = wt.join("target").join(relative);
2573 let file = OpenOptions::new()
2574 .write(true)
2575 .create(true)
2576 .truncate(false)
2577 .open(path)
2578 .unwrap();
2579 let at = if evidence == "future" {
2580 now + jiff::SignedDuration::from_secs(10)
2581 } else {
2582 recent
2583 };
2584 file.set_modified(std::time::SystemTime::from(at)).unwrap();
2585 }
2586 }
2587 let mut disk = Fx::disk();
2588 disk.fold_grace_secs = 3600;
2589 let report = sweep(&f.home, &disk, now).await;
2590 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2591 assert!(
2592 report.retained.iter().any(|s| s.contains("grace period")
2593 || s.contains("execution activity after inventory")),
2594 "{evidence}: {report:?}"
2595 );
2596 let aged = now + jiff::SignedDuration::from_secs(7200);
2597 let report = sweep(&f.home, &disk, aged).await;
2598 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2599 }
2600 }
2601
2602 #[tokio::test]
2603 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2604 let f = Fx::new().await;
2605 let state = f.run("ac02", true).await;
2606 let wt = &state.candidates[0].worktree;
2607 missing_tag_target(wt);
2608 let target = wt.join("target");
2609 let unknown = target.join("debug/notes");
2610 std::fs::write(&unknown, "operator data").unwrap();
2611 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2612 let path = target.join("debug").join(lock);
2613 let file = OpenOptions::new()
2614 .create(true)
2615 .truncate(false)
2616 .read(true)
2617 .write(true)
2618 .open(&path)
2619 .unwrap();
2620 crate::pr_head::try_lock_file(&file).unwrap();
2621 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2622 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2623 assert!(
2624 report
2625 .retained
2626 .iter()
2627 .any(|s| s.contains(path.to_str().unwrap())),
2628 "{report:?}"
2629 );
2630 }
2631 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2632 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2633 drop(owner);
2634 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2635 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2636 drop(driver);
2637 let output = OutputLease::borrow(wt).unwrap().unwrap();
2638 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2639 drop(output);
2640 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2641 assert_eq!(report.freed_bytes, 200, "{report:?}");
2642 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2643 assert!(!target.join("CACHEDIR.TAG").exists());
2644 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2645 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2646 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2647 }
2648
2649 #[tokio::test]
2650 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2651 let f = Fx::new().await;
2652 let state = f.run("ac04", true).await;
2653 let wt = &state.candidates[0].worktree;
2654 missing_tag_target(wt);
2655 let target = wt.join("target");
2656 let first = target.join("debug/output");
2657 let second = target.join("debug/second");
2658 std::fs::copy(&first, &second).unwrap();
2659 std::fs::write(
2660 second.with_extension("d"),
2661 format!("{}: source\n", second.display()),
2662 )
2663 .unwrap();
2664 let outputs = declared_outputs(&target).unwrap();
2665 assert_eq!(outputs.len(), 2);
2666 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2668 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2669 std::fs::remove_file(first).unwrap();
2670 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2671 assert_eq!(report.freed_bytes, 200, "{report:?}");
2672 assert!(!second.exists());
2673 assert!(second.with_extension("d").exists());
2674 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2675 }
2676
2677 #[tokio::test]
2678 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2679 for invalid in [
2680 "identity",
2681 "deps",
2682 "fingerprint",
2683 "lock",
2684 "dep-info",
2685 "tag-symlink",
2686 "bad-tag",
2687 ] {
2688 let f = Fx::new().await;
2689 let state = f.run("ac03", true).await;
2690 let wt = &state.candidates[0].worktree;
2691 missing_tag_target(wt);
2692 let target = wt.join("target");
2693 match invalid {
2694 "identity" => std::fs::write(
2695 target.join(".rustc_info.json"),
2696 r#"{"rustc_fingerprint":123}"#,
2697 )
2698 .unwrap(),
2699 "deps" | "fingerprint" => {
2700 let path = target.join("debug").join(if invalid == "deps" {
2701 "deps"
2702 } else {
2703 ".fingerprint"
2704 });
2705 std::fs::remove_dir(&path).unwrap();
2706 std::os::unix::fs::symlink(&f.repo, path).unwrap();
2707 }
2708 "lock" => {
2709 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2710 }
2711 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2712 "tag-symlink" => {
2713 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2714 .unwrap()
2715 }
2716 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2717 _ => unreachable!(),
2718 }
2719 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2720 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2721 assert!(target.join("debug/output").is_file());
2722 }
2723 }
2724
2725 #[tokio::test]
2726 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2727 let f = Fx::new().await;
2728 let first = f.run("ba00", true).await;
2729 let second = f.run("ba01", true).await;
2730 for state in [&first, &second] {
2731 Fx::target(&state.candidates[0].worktree);
2732 }
2733 let mut disk = Fx::disk();
2734 disk.worktree_reclaim_bytes = 100;
2735 let out = sweep(&f.home, &disk, Fx::now()).await;
2736 assert_eq!(out.freed_bytes, 200, "{out:?}");
2737 assert!(
2738 out.retained
2739 .contains(&format!("{}: per-pass byte budget spent", second.id)),
2740 "{out:?}"
2741 );
2742 assert!(
2743 !first.candidates[0]
2744 .worktree
2745 .join("target/debug/output")
2746 .exists()
2747 );
2748 assert!(
2749 second.candidates[0]
2750 .worktree
2751 .join("target/debug/output")
2752 .exists()
2753 );
2754 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2755 }
2756
2757 #[tokio::test]
2758 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2759 let f = Fx::new().await;
2760 let small = f.run("bb00", true).await;
2761 let large = f.run("bb01", true).await;
2762 for state in [&small, &large] {
2763 Fx::target(&state.candidates[0].worktree);
2764 }
2765 let output = large.candidates[0].worktree.join("target/debug/output");
2766 std::fs::OpenOptions::new()
2767 .write(true)
2768 .open(&output)
2769 .unwrap()
2770 .set_len(1000)
2771 .unwrap();
2772 let mut disk = Fx::disk();
2773 disk.worktree_reclaim_bytes = 300;
2774 disk.min_free_bytes = u64::MAX;
2775 disk.worktree_limit_bytes = u64::MAX;
2777 let out = sweep(&f.home, &disk, Fx::now()).await;
2778 assert_eq!(out.freed_bytes, 1000, "{out:?}");
2779 assert!(!output.exists());
2780 assert!(
2781 small.candidates[0]
2782 .worktree
2783 .join("target/debug/output")
2784 .exists()
2785 );
2786 }
2787
2788 #[tokio::test]
2789 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2790 let f = Fx::new().await;
2791 let first = f.run("bc00", true).await;
2792 let second = f.run("bc01", true).await;
2793 for state in [&first, &second] {
2794 Fx::target(&state.candidates[0].worktree);
2795 }
2796 std::fs::create_dir_all(
2798 first.candidates[0]
2799 .worktree
2800 .join("target/debug/incremental"),
2801 )
2802 .unwrap();
2803 std::fs::write(
2804 first.candidates[0]
2805 .worktree
2806 .join("target/debug/incremental/cache"),
2807 [0; 1000],
2808 )
2809 .unwrap();
2810 let mut disk = Fx::disk();
2811 disk.worktree_reclaim_bytes = 0;
2812 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2813 disk.worktree_reclaim_bytes = 300;
2814 let out = sweep(&f.home, &disk, Fx::now()).await;
2815 assert_eq!(out.freed_bytes, 200, "{out:?}");
2816 assert!(
2817 !first.candidates[0]
2818 .worktree
2819 .join("target/debug/output")
2820 .exists()
2821 );
2822 assert!(
2823 second.candidates[0]
2824 .worktree
2825 .join("target/debug/output")
2826 .exists()
2827 );
2828 assert!(
2829 out.retained
2830 .iter()
2831 .any(|r| r.contains("declared outputs exceed"))
2832 );
2833 }
2834
2835 #[tokio::test]
2836 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2837 let f = Fx::new().await;
2838 let mut protected = f.run("bd00", true).await;
2839 let later = f.run("bd01", true).await;
2840 for state in [&protected, &later] {
2841 Fx::target(&state.candidates[0].worktree);
2842 }
2843 let mut disk = Fx::disk();
2844 disk.worktree_reclaim_bytes = 100;
2845 disk.fold_grace_secs = 3600;
2846 let out = sweep(&f.home, &disk, Fx::now()).await;
2847 assert_eq!(out.freed_bytes, 0);
2848 assert!(
2849 out.retained
2850 .iter()
2851 .any(|r| r.contains("within grace period"))
2852 );
2853 disk.fold_grace_secs = 0;
2854 protected.driver_exited = false;
2855 protected.driver_pid = Some(std::process::id());
2856 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2857 protected
2858 .save_under_at(&f.home, protected.updated_at)
2859 .unwrap();
2860 let out = sweep(&f.home, &disk, Fx::now()).await;
2861 assert!(
2862 out.retained
2863 .iter()
2864 .any(|r| r.contains("run live or ownership unknown"))
2865 );
2866 assert_eq!(
2867 out.freed_bytes, 200,
2868 "live target must not consume exception: {out:?}"
2869 );
2870 let output = protected.candidates[0].worktree.join("target/debug/output");
2871 assert!(output.exists());
2872 protected.driver_exited = true;
2873 protected.save_under(&f.home).unwrap();
2874 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2875 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2876 assert!(output.exists());
2877 drop(owner);
2878 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2879 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2880 assert!(output.exists());
2881 drop(cargo);
2882 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2883 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2884 assert!(output.exists());
2885 drop(lease);
2886 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2887 assert!(!output.exists());
2888 }
2889
2890 #[tokio::test]
2891 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2892 for remove_root in [true, false] {
2893 let f = Fx::new().await;
2894 let stale = Fx::new().await;
2895 let mut missing = stale.run("aa00", false).await;
2896 Fx::target(&missing.candidates[0].worktree);
2897 missing.save_under(&f.home).unwrap();
2898 if remove_root {
2899 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2900 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2903 } else {
2904 std::fs::remove_dir_all(&stale.repo).unwrap();
2905 }
2906 let healthy = f.run("aa01", false).await;
2907 Fx::target(&healthy.candidates[0].worktree);
2908
2909 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2910 assert!(report.failures.is_empty(), "{report:?}");
2911 assert_eq!(report.freed_bytes, 200, "{report:?}");
2912 assert!(
2913 !healthy.candidates[0]
2914 .worktree
2915 .join("target/debug/output")
2916 .exists()
2917 );
2918 if !remove_root {
2919 assert!(
2921 missing.candidates[0]
2922 .worktree
2923 .join("target/debug/output")
2924 .exists()
2925 );
2926 }
2927 }
2928 }
2929
2930 #[tokio::test]
2931 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2932 let f = Fx::new().await;
2933 let mut missing = f.run("aa00", false).await;
2934 Fx::target(&missing.candidates[0].worktree);
2935 missing.repo = missing.candidates[0].worktree.clone();
2936 missing.save_under(&f.home).unwrap();
2937 std::fs::remove_dir_all(&missing.repo).unwrap();
2938 assert!(missing.worktree_root().is_dir());
2939 assert!(f.repo.is_dir());
2940
2941 let healthy = f.run("aa01", false).await;
2942 Fx::target(&healthy.candidates[0].worktree);
2943
2944 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2945 assert!(report.failures.is_empty(), "{report:?}");
2946 assert_eq!(report.freed_bytes, 200, "{report:?}");
2947 assert!(
2948 !healthy.candidates[0]
2949 .worktree
2950 .join("target/debug/output")
2951 .exists()
2952 );
2953 }
2954
2955 #[tokio::test]
2956 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2957 let f = Fx::new().await;
2958 let broken = Fx::new().await;
2959 let mut invalid = broken.run("aa00", false).await;
2960 Fx::target(&invalid.candidates[0].worktree);
2961 invalid.save_under(&f.home).unwrap();
2962 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2963 let healthy = f.run("aa01", false).await;
2964 Fx::target(&healthy.candidates[0].worktree);
2965
2966 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2967 assert_eq!(report.failures.len(), 1, "{report:?}");
2968 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2969 assert_eq!(report.freed_bytes, 200, "{report:?}");
2970 assert!(
2971 !healthy.candidates[0]
2972 .worktree
2973 .join("target/debug/output")
2974 .exists()
2975 );
2976 assert!(
2977 invalid.candidates[0]
2978 .worktree
2979 .join("target/debug/output")
2980 .exists()
2981 );
2982 }
2983
2984 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2985 let path = reconcile::run(&f.home, &task.id, &source.id, None)
2986 .await
2987 .unwrap();
2988 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2989 }
2990
2991 #[tokio::test]
2992 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
2993 use crate::{
2994 chat_control::{Operation, dispatch},
2995 talk::{self, Talks},
2996 };
2997 let f = Fx::new().await;
2998 let source = f.run("ae00", true).await;
2999 let clean = f.run("ae01", false).await;
3000 let queue = crate::queue::Queue::at(f.home.join("queue"));
3001 let talks = Talks::at(f.home.join("talks"));
3002 let open_talk = |id: &str| -> talk::Talk {
3003 let mut conversation = serde_json::from_value(serde_json::json!({
3004 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
3005 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
3006 "seat":crate::agent::SeatState::new("talk","worker",1)
3007 }))
3008 .unwrap();
3009 talks.put(&mut conversation).unwrap();
3010 conversation
3011 };
3012 let conversation = open_talk("talk-owner");
3013 let other = open_talk("talk-other");
3014 let task = f.task(&[source.clone(), clean.clone()]);
3016 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3017 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
3018 task: task.id.clone(),
3019 authority: source.id.clone(),
3020 apply: apply.map(str::to_owned),
3021 };
3022 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
3024 .await
3025 .unwrap();
3026 let legacy: serde_json::Value =
3027 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
3028 let legacy_id = legacy["id"].as_str().unwrap();
3029 assert!(
3030 dispatch(
3031 &queue,
3032 &f.home,
3033 &talks,
3034 &conversation.id,
3035 operation(Some(legacy_id))
3036 )
3037 .await
3038 .is_err()
3039 );
3040 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
3041 .await
3042 .unwrap();
3043 assert!(summary.contains("Complete review summary"));
3044 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
3045 let root = f.home.join("retention-reconciliations");
3046 let displayed: serde_json::Value =
3047 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
3048 let id = displayed["plan"].as_str().unwrap();
3049 let dir = root.join(id);
3050 let plan: serde_json::Value =
3051 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
3052 assert_eq!(plan["version"], 2);
3053 assert!(clean.candidates[0].worktree.exists());
3054 assert!(!dir.join("approved.json").exists());
3055 assert!(
3057 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
3058 .await
3059 .is_err()
3060 );
3061 assert!(!dir.join("approved.json").exists());
3062 let error = reconcile::interrupt_chat_after_fence(
3065 &f.home,
3066 &task,
3067 id,
3068 reconcile::ChatRequest {
3069 talks: &talks,
3070 conversation: &conversation.id,
3071 },
3072 )
3073 .await
3074 .unwrap_err();
3075 assert!(
3076 error.to_string().contains("simulated interruption"),
3077 "{error:#}"
3078 );
3079 assert!(clean.candidates[0].worktree.exists());
3080 assert!(dir.join("approved.json").exists());
3081 dispatch(
3082 &queue,
3083 &f.home,
3084 &talks,
3085 &conversation.id,
3086 operation(Some(id)),
3087 )
3088 .await
3089 .unwrap();
3090 assert!(!clean.candidates[0].worktree.exists());
3091 assert!(source.candidates[0].worktree.exists());
3092 let audit: serde_json::Value =
3093 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3094 assert_eq!(audit["version"], 2);
3095 assert_eq!(
3096 audit["owner"],
3097 serde_json::json!({"conversation": conversation.id})
3098 );
3099 dispatch(
3100 &queue,
3101 &f.home,
3102 &talks,
3103 &conversation.id,
3104 operation(Some(id)),
3105 )
3106 .await
3107 .unwrap();
3108 }
3109
3110 #[tokio::test]
3111 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3112 let f = Fx::new().await;
3113 let source = f.run("af00", true).await;
3114 let mut clean = f.run("af01", false).await;
3115 let mut dirty = f.run("af02", false).await;
3116 for state in [&mut clean, &mut dirty] {
3117 state.status = RunStatus::Prep;
3118 state.prep_complete = Some(false);
3119 state.candidates[0].empty = false;
3120 state.save_under(&f.home).unwrap();
3121 }
3122 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3123 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3124 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3125 let plan = reconciliation_plan(&f, &task, &source).await;
3126 let selected: Vec<_> = plan["candidates"]
3127 .as_array()
3128 .unwrap()
3129 .iter()
3130 .filter(|e| e["selected"] == true)
3131 .collect();
3132 assert_eq!(selected.len(), 1, "{plan:#}");
3133 assert_eq!(selected[0]["run"], clean.id);
3134 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3135 .await
3136 .unwrap();
3137 assert!(!clean.candidates[0].worktree.exists());
3138 assert!(dirty.candidates[0].worktree.exists());
3139 assert!(
3140 RunState::load_under(&dirty.id, &f.home)
3141 .unwrap()
3142 .retired_to
3143 .is_none()
3144 );
3145 }
3146
3147 #[tokio::test]
3148 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3149 let f = Fx::new().await;
3150 let mut source = f.run("aa00", true).await;
3151 let alternative = f.run("aa01", true).await;
3152 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3153 .await
3154 .unwrap();
3155 source.reviewed_commits = Some(vec![head.clone()]);
3156 source.save_under(&f.home).unwrap();
3157 let mut runs = vec![source.clone(), alternative.clone()];
3158 for i in 2..14 {
3159 runs.push(f.run(&format!("aa{i:02}"), false).await);
3160 }
3161 let mut task = f.task(&runs);
3162 task.pr_recovery = Some(crate::pr_recovery::Selection {
3163 url: "https://example.invalid/pull/1".into(),
3164 branch: "selected".into(),
3165 head: head.clone(),
3166 source_head: Some(head.clone()),
3167 source_run: Some(source.id.clone()),
3168 source_proof: None,
3169 disposition: Some(crate::pr_recovery::Disposition::Land),
3170 instruction: task.instruction.clone(),
3171 approved_at: Timestamp::now(),
3172 });
3173 source.pr_recovery = task.pr_recovery.clone();
3174 source.save_under(&f.home).unwrap();
3175 runs[0] = source.clone();
3176 crate::queue::Queue::at(f.home.join("queue"))
3177 .put(&mut task)
3178 .unwrap();
3179 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3180 let task_before = serde_json::to_value(&task).unwrap();
3181 let mut report = Report::default();
3182 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3183 assert_eq!(
3184 report.retired, 0,
3185 "automatic cleanup must still refuse blanket markers"
3186 );
3187 assert!(
3188 report
3189 .retained
3190 .iter()
3191 .any(|reason| reason.contains("explicitly retained"))
3192 );
3193 let mut total = 0;
3194 for expected in [5, 5, 2, 0] {
3195 let plan = reconciliation_plan(&f, &task, &source).await;
3196 let selected: Vec<_> = plan["candidates"]
3197 .as_array()
3198 .unwrap()
3199 .iter()
3200 .filter(|e| e["selected"] == true)
3201 .collect();
3202 assert_eq!(selected.len(), expected, "{plan:#}");
3203 for e in &selected {
3204 assert!(
3205 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3206 .unwrap()
3207 .retired_to
3208 .is_none()
3209 );
3210 }
3211 let operation = plan["id"].as_str().unwrap();
3212 for _ in 0..2 {
3213 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3214 .await
3215 .unwrap();
3216 }
3217 total += expected;
3218 }
3219 assert_eq!(total, 12);
3220 for run in &runs {
3221 let dir = f.home.join("runs").join(&run.id);
3222 assert!(dir.join("run.json").exists());
3223 assert!(dir.join("retain-recovery-sources").exists());
3224 assert!(
3225 git::branch_exists(&f.repo, &run.candidates[0].branch)
3226 .await
3227 .unwrap()
3228 );
3229 }
3230 assert!(source.candidates[0].worktree.exists());
3231 assert!(alternative.candidates[0].worktree.exists());
3232 assert_eq!(
3233 git::rev_parse(&f.repo, &source.candidates[0].branch)
3234 .await
3235 .unwrap(),
3236 head
3237 );
3238 assert_eq!(
3239 serde_json::to_value(
3240 crate::queue::Queue::at(f.home.join("queue"))
3241 .get(&task.id)
3242 .unwrap()
3243 )
3244 .unwrap(),
3245 task_before
3246 );
3247 assert_eq!(
3248 runs.iter()
3249 .filter(|r| RunState::load_under(&r.id, &f.home)
3250 .unwrap()
3251 .retired_to
3252 .is_none())
3253 .count(),
3254 2
3255 );
3256 }
3257
3258 #[tokio::test]
3259 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3260 let f = Fx::new().await;
3261 let source = f.run("ab00", true).await;
3262 let mut runs = vec![source.clone()];
3263 for i in 1..9 {
3264 runs.push(f.run(&format!("ab{i:02}"), false).await);
3265 }
3266 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3267 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3268 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3269 std::fs::write(
3270 runs[3].candidates[0].worktree.join("scratch/source"),
3271 "ignored source",
3272 )
3273 .unwrap();
3274 std::fs::write(
3275 runs[4].worktree_root().join("unknown"),
3276 "unregistered source",
3277 )
3278 .unwrap();
3279 let checkpoint = git::git(
3280 &runs[5].candidates[0].worktree,
3281 &[
3282 "rev-parse",
3283 "--path-format=absolute",
3284 "--git-path",
3285 "sequencer",
3286 ],
3287 )
3288 .await
3289 .unwrap();
3290 std::fs::create_dir(checkpoint.trim()).unwrap();
3291 runs[6].driver_exited = false;
3292 runs[6].driver_pid = Some(std::process::id());
3293 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3294 runs[6].save_under(&f.home).unwrap();
3295 runs[7].driver_exited = false;
3296 runs[7].driver_pid = None;
3297 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3299 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3300
3301 let hidden = &runs[8].candidates[0].worktree;
3302 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3303 .await
3304 .unwrap();
3305 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3306 assert!(
3307 git::git(hidden, &["status", "--porcelain"])
3308 .await
3309 .unwrap()
3310 .is_empty()
3311 );
3312 let task = f.task(&runs);
3313 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3314 let plan = reconciliation_plan(&f, &task, &source).await;
3315 assert!(
3316 plan["candidates"]
3317 .as_array()
3318 .unwrap()
3319 .iter()
3320 .all(|e| e["selected"] == false),
3321 "{plan:#}"
3322 );
3323 for run in &runs {
3324 assert!(run.candidates[0].worktree.exists());
3325 }
3326 }
3327
3328 #[tokio::test]
3329 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3330 let f = Fx::new().await;
3331 let source = f.run("ac00", true).await;
3332 let empty = f.run("ac01", false).await;
3333 let task = f.task(&[source.clone(), empty.clone()]);
3334 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3335 let plan = reconciliation_plan(&f, &task, &source).await;
3336 std::fs::write(
3337 f.home.join("runs").join(&source.id).join("new-report"),
3338 "new review checkpoint",
3339 )
3340 .unwrap();
3341 assert!(
3342 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3343 .await
3344 .is_err()
3345 );
3346 assert!(empty.candidates[0].worktree.exists());
3347 assert!(
3348 RunState::load_under(&empty.id, &f.home)
3349 .unwrap()
3350 .retired_to
3351 .is_none()
3352 );
3353 let marker = f
3354 .home
3355 .join("runs")
3356 .join(&empty.id)
3357 .join("retain-recovery-sources");
3358 std::fs::write(&marker, "different owner").unwrap();
3359 let plan = reconciliation_plan(&f, &task, &source).await;
3360 assert!(
3361 plan["candidates"]
3362 .as_array()
3363 .unwrap()
3364 .iter()
3365 .all(|e| e["selected"] == false)
3366 );
3367 }
3368
3369 #[tokio::test]
3370 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3371 for partial in [false, true] {
3372 let f = Fx::new().await;
3373 let source = f.run("ab10", true).await;
3374 let first = f.run("ab11", false).await;
3375 let second = f.run("ab12", false).await;
3376 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3377 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3378 let plan = reconciliation_plan(&f, &task, &source).await;
3379 let id = plan["id"].as_str().unwrap();
3380 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3381 std::fs::write(&staging, b"").unwrap();
3383 assert!(
3384 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3385 .await
3386 .is_err()
3387 );
3388 assert!(staging.exists());
3389 std::fs::remove_file(&staging).unwrap();
3390 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3391 .await
3392 .unwrap_err();
3393 assert!(
3394 error.to_string().contains("before run state rename"),
3395 "{error:#}"
3396 );
3397 assert!(
3398 RunState::load_under(&first.id, &f.home)
3399 .unwrap()
3400 .retired_to
3401 .is_some()
3402 );
3403 assert!(
3404 RunState::load_under(&second.id, &f.home)
3405 .unwrap()
3406 .retired_to
3407 .is_none()
3408 );
3409 assert!(first.candidates[0].worktree.exists());
3410 assert!(second.candidates[0].worktree.exists());
3411 let interrupted = std::fs::read(&staging).unwrap();
3412 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3414 assert!(
3415 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3416 .await
3417 .is_err()
3418 );
3419 assert_eq!(
3420 std::fs::read(&staging).unwrap(),
3421 b"unrelated recovery source"
3422 );
3423 std::fs::write(&staging, interrupted).unwrap();
3424 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3425 .await
3426 .unwrap();
3427 assert!(!staging.exists());
3428 assert!(!first.candidates[0].worktree.exists());
3429 assert!(!second.candidates[0].worktree.exists());
3430 assert_eq!(
3431 RunState::load_under(&second.id, &f.home)
3432 .unwrap()
3433 .retired_to,
3434 Some(source.id)
3435 );
3436 }
3437 }
3438
3439 #[tokio::test]
3440 async fn reconciliation_resumes_only_its_exact_durable_fence() {
3441 let f = Fx::new().await;
3442 let source = f.run("ad00", true).await;
3443 let empty = f.run("ad01", false).await;
3444 let task = f.task(&[source.clone(), empty.clone()]);
3445 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3446 let plan = reconciliation_plan(&f, &task, &source).await;
3447 let id = plan["id"].as_str().unwrap();
3448 assert!(
3449 reconcile::interrupt_after_fence(&f.home, &task, id)
3450 .await
3451 .is_err()
3452 );
3453 assert!(empty.candidates[0].worktree.exists());
3454 assert_eq!(
3455 RunState::load_under(&empty.id, &f.home)
3456 .unwrap()
3457 .retired_to
3458 .as_ref(),
3459 Some(&source.id)
3460 );
3461 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3462 .await
3463 .unwrap();
3464 assert!(!empty.candidates[0].worktree.exists());
3465 git::worktree_add_branch(
3467 &f.repo,
3468 &empty.candidates[0].worktree,
3469 "restore-empty",
3470 &f.base,
3471 )
3472 .await
3473 .unwrap();
3474 assert!(
3476 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3477 .await
3478 .is_err()
3479 );
3480 assert!(empty.candidates[0].worktree.exists());
3481 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3482 changed.prep_failure = Some("new owner edit".into());
3483 changed.save_under(&f.home).unwrap();
3484 assert!(
3485 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3486 .await
3487 .is_err()
3488 );
3489 }
3490
3491 #[tokio::test]
3492 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3493 let f = Fx::new().await;
3494 let source = f.run("ae00", true).await;
3495 let mut prep = f.run("ae01", false).await;
3496 let candidate = prep.candidates.pop().unwrap();
3497 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3498 prep.save_under(&f.home).unwrap();
3499 let reset = f.run("ae02", false).await;
3500 let wt = &reset.candidates[0].worktree;
3501 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3502 git::git(wt, &["add", "."]).await.unwrap();
3503 git::git(wt, &["commit", "-m", "unique implementation"])
3504 .await
3505 .unwrap();
3506 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3507 let unknown = f.run("ae03", false).await;
3508 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3509 let mut raw: serde_json::Value =
3510 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3511 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3512 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3513 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3514 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3515 let plan = reconciliation_plan(&f, &task, &source).await;
3516 let selected: Vec<_> = plan["candidates"]
3517 .as_array()
3518 .unwrap()
3519 .iter()
3520 .filter(|e| e["selected"] == true)
3521 .collect();
3522 assert_eq!(selected.len(), 1, "{plan:#}");
3523 assert_eq!(selected[0]["run"], prep.id);
3524 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3525 .await
3526 .unwrap();
3527 assert!(!candidate.worktree.exists());
3528 assert!(reset.candidates[0].worktree.exists());
3529 assert!(unknown.candidates[0].worktree.exists());
3530 }
3531
3532 #[tokio::test]
3533 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3534 let f = Fx::new().await;
3535 let mut state = f.run("aaaa", true).await;
3536 let candidate = state.candidates[0].worktree.clone();
3537 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3538 Fx::target(&candidate);
3539 for seat in ["review-1", "review-2"] {
3540 let wt = state.worktree_root().join(seat);
3541 git::worktree_add_detached(&f.repo, &wt, &f.base)
3542 .await
3543 .unwrap();
3544 Fx::target(&wt);
3545 }
3546 state.status = RunStatus::Landing;
3547 state.save_under(&f.home).unwrap();
3548 let second = Fx::new().await;
3549 let other = second.run("bbbb", true).await;
3550 let mut imported = other.clone();
3552 imported.save_under(&f.home).unwrap();
3553 Fx::target(&other.candidates[0].worktree);
3554 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3555 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3556 assert!(
3557 candidate.join("target/debug/output").exists(),
3558 "active writer protected"
3559 );
3560 assert!(
3561 !state
3562 .worktree_root()
3563 .join("review-1/target/debug/output")
3564 .exists()
3565 );
3566 assert!(
3567 !state
3568 .worktree_root()
3569 .join("review-2/target/debug/output")
3570 .exists()
3571 );
3572 assert!(
3573 !other.candidates[0]
3574 .worktree
3575 .join("target/debug/output")
3576 .exists()
3577 );
3578 assert!(out.freed_bytes >= 600);
3579 drop(guard);
3580 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3581 assert!(out.freed_bytes >= 200);
3582 assert_eq!(
3583 std::fs::read_to_string(candidate.join("source")).unwrap(),
3584 "uncommitted work"
3585 );
3586 assert!(
3587 git::branch_exists(&f.repo, &state.candidates[0].branch)
3588 .await
3589 .unwrap()
3590 );
3591 }
3592
3593 #[tokio::test]
3594 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3595 let f = Fx::new().await;
3596 let state = f.run("cc01", false).await;
3597 let wt = &state.candidates[0].worktree;
3598 Fx::target(wt);
3599 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3600 let dir = wt.join("target").join(dir);
3601 std::fs::create_dir_all(&dir).unwrap();
3602 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3603 }
3604 let manifest = wt.join("target/debug/output.d");
3606 let output = wt.join("target/debug/output");
3607 std::fs::write(
3608 &manifest,
3609 format!(
3610 "{}: {}\n",
3611 output.display(),
3612 wt.join("target/debug/notes.txt").display()
3613 ),
3614 )
3615 .unwrap();
3616 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3617 assert_eq!(report.freed_bytes, 200);
3618 assert!(!output.exists());
3619 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3620 assert_eq!(
3621 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3622 "user evidence"
3623 );
3624 }
3625 assert!(manifest.exists());
3627 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3628 std::fs::write(&output, "user note replacing an old binary").unwrap();
3629 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3630 assert_eq!(report.freed_bytes, 0);
3631 assert_eq!(
3632 std::fs::read_to_string(&output).unwrap(),
3633 "user note replacing an old binary"
3634 );
3635 }
3636
3637 #[tokio::test]
3638 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3639 let f = Fx::new().await;
3640 let state = f.run("cc02", false).await;
3641 let wt = &state.candidates[0].worktree;
3642 Fx::target(wt);
3643 let mut disk = Fx::disk();
3644 disk.worktree_limit_bytes = u64::MAX;
3645 let now = Fx::now();
3646 let first = sweep(&f.home, &disk, now).await;
3647 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3648 assert_eq!(
3649 sweep(&f.home, &disk, now).await.measured_bytes,
3650 first.measured_bytes
3651 );
3652 assert!(
3653 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3654 .await
3655 .measured_bytes
3656 > first.measured_bytes
3657 );
3658 disk.worktree_limit_bytes = 0;
3659 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3660 }
3661
3662 #[tokio::test]
3663 async fn unfinished_source_fold_reports_its_retention_reason() {
3664 let f = Fx::new().await;
3665 let mut source = f.run("cc03", true).await;
3666 let task = f.task(&[source.clone()]);
3667 let error = crate::graph::fold_run(&mut source, true, &f.home)
3668 .await
3669 .unwrap_err();
3670 assert!(error.to_string().contains(&task.id));
3671 assert!(source.candidates[0].worktree.exists());
3672 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3673 let error = crate::graph::fold_run(&mut source, true, &f.home)
3674 .await
3675 .unwrap_err();
3676 assert!(error.to_string().contains("explicitly retained"));
3677 }
3678
3679 #[tokio::test]
3680 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3681 let f = Fx::new().await;
3682 let mut source = f.run("cc04", true).await;
3683 let mut task = f.task(&[source.clone()]);
3684 task.hold_manual(Some("queue settlement pending".into()));
3685 let queue = crate::queue::Queue::at(f.home.join("queue"));
3686 queue.put(&mut task).unwrap();
3687 source.merge = Some(crate::run::MergeOutcome {
3688 mode: crate::config::MergeMode::Pr,
3689 ok: true,
3690 detail: "confirmed merged".into(),
3691 empty: false,
3692 });
3693 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3695 source.status = status;
3696 assert!(
3697 crate::graph::fold_run(&mut source, true, &f.home)
3698 .await
3699 .is_err()
3700 );
3701 assert!(source.candidates[0].worktree.exists());
3702 }
3703 git::git(
3704 &f.repo,
3705 &["merge", "--ff-only", &source.candidates[0].branch],
3706 )
3707 .await
3708 .unwrap();
3709 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3710 .await
3711 .unwrap();
3712 source.status = RunStatus::Merged;
3713 source.save_under(&f.home).unwrap();
3714 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3715 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3716 assert!(
3717 crate::graph::fold_run(&mut source, true, &f.home)
3718 .await
3719 .unwrap_err()
3720 .to_string()
3721 .contains("explicitly retained")
3722 );
3723 std::fs::remove_file(
3726 f.home
3727 .join("runs")
3728 .join(&source.id)
3729 .join("retain-recovery-sources"),
3730 )
3731 .unwrap();
3732 let (folded, unreadable) = crate::clean::fold_due(
3733 &f.home.join("runs"),
3734 &f.home,
3735 &f.bay,
3736 &Fx::disk(),
3737 Fx::now(),
3738 )
3739 .await
3740 .unwrap();
3741 assert_eq!((folded, unreadable), (1, 0));
3742 assert!(!source.candidates[0].worktree.exists());
3743 let evidence = f
3744 .home
3745 .join("runs")
3746 .join(&source.id)
3747 .join("artifacts/fold-evidence");
3748 let names: Vec<_> = std::fs::read_dir(evidence)
3749 .unwrap()
3750 .map(|entry| entry.unwrap().file_name())
3751 .collect();
3752 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3753 assert_eq!(
3754 queue.get(&task.id).unwrap().status,
3755 crate::queue::TaskStatus::Held
3756 );
3757 }
3758
3759 #[tokio::test]
3760 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3761 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3762 let f = Fx::new().await;
3763 let mut state = f.run("cc06", true).await;
3764 let candidate = state.candidates[0].clone();
3765 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3766 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3767 git::rescue_commit(&candidate.worktree, "second candidate change")
3768 .await
3769 .unwrap();
3770 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3771 state.tally = Some(crate::run::Tally {
3772 first_choice: Default::default(),
3773 borda: Default::default(),
3774 winner: 'A',
3775 rankings: 0,
3776 unanimous_initial: true,
3777 deliberated: false,
3778 changed_votes: 0,
3779 unanimous_final: true,
3780 tie_break: None,
3781 judges: 0,
3782 present: 0,
3783 quorum: 0,
3784 met_quorum: true,
3785 uncontested: Some("fixture".into()),
3786 });
3787 let reviewer = state.worktree_root().join("review-1");
3790 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3791 .await
3792 .unwrap();
3793 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3796 git::rescue_commit(&f.repo, "advance before squash")
3797 .await
3798 .unwrap();
3799 let published = if variant == "partial" {
3800 &intermediate
3801 } else {
3802 &head
3803 };
3804 git::git(&f.repo, &["merge", "--squash", published])
3805 .await
3806 .unwrap();
3807 git::git(&f.repo, &["commit", "-m", "squash candidate"])
3808 .await
3809 .unwrap();
3810 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3811 assert!(
3812 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3813 .await
3814 .unwrap()
3815 .ok()
3816 );
3817 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3818 git::rescue_commit(&f.repo, "advance after squash")
3819 .await
3820 .unwrap();
3821 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3822 .await
3823 .unwrap();
3824 if matches!(variant, "checkpoint" | "reviewer") {
3825 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3827 git::rescue_commit(&reviewer, "unpublished checkpoint")
3828 .await
3829 .unwrap();
3830 if variant == "checkpoint" {
3831 git::git(&reviewer, &["reset", "--hard", &intermediate])
3832 .await
3833 .unwrap();
3834 }
3835 }
3836 if variant == "loser" {
3837 let mut loser = candidate.clone();
3838 loser.label = 'B';
3839 loser.branch = "magi/cc06/B".into();
3840 loser.worktree = state.worktree_root().join("cand-B");
3841 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3842 .await
3843 .unwrap();
3844 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3845 git::rescue_commit(&loser.worktree, "independent loser")
3846 .await
3847 .unwrap();
3848 state.candidates.push(loser);
3849 }
3850 state.status = RunStatus::Merged;
3851 state.merge = Some(crate::run::MergeOutcome {
3852 mode: crate::config::MergeMode::Pr,
3853 ok: true,
3854 detail: "candidate A squash merged".into(),
3855 empty: false,
3856 });
3857 state.save_under(&f.home).unwrap();
3858 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3859 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3860 result.unwrap();
3861 assert!(
3862 !candidate.worktree.exists() && !reviewer.exists(),
3863 "{variant}"
3864 );
3865 assert!(
3866 !git::branch_exists(&f.repo, &candidate.branch)
3867 .await
3868 .unwrap()
3869 );
3870 let artifacts = f
3871 .home
3872 .join("runs")
3873 .join(&state.id)
3874 .join("artifacts/fold-evidence");
3875 assert_eq!(
3876 artifacts.join(format!("{head}.patch")).exists(),
3877 variant == "partial",
3878 "only a partially published winner needs evidence: {variant}"
3879 );
3880 assert_eq!(
3881 artifacts.join(format!("{intermediate}.patch")).exists(),
3882 variant == "partial",
3883 "squash proof also covers intermediate reviewer tips: {variant}"
3884 );
3885 let patches: Vec<_> = std::fs::read_dir(&artifacts)
3886 .unwrap()
3887 .map(|entry| entry.unwrap().path())
3888 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3889 .collect();
3890 let expected = match variant {
3891 "partial" => 2,
3892 "loser" | "reviewer" => 1,
3893 _ => 0,
3894 };
3895 assert_eq!(patches.len(), expected, "{variant}");
3896 if variant == "loser" {
3897 assert!(
3898 std::fs::read_to_string(&patches[0])
3899 .unwrap()
3900 .contains("losing candidate")
3901 );
3902 }
3903 assert!(artifacts.join("manifest.json").exists());
3904 if variant == "reviewer" {
3905 assert!(
3906 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3907 .unwrap()
3908 .contains("reviewer checkpoint")
3909 );
3910 }
3911 crate::graph::fold_run(&mut state, true, &f.home)
3912 .await
3913 .unwrap();
3914 }
3915 }
3916
3917 #[tokio::test]
3918 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3919 for missing in [true, false] {
3920 let f = Fx::new().await;
3921 let mut state = f.run("cc05", true).await;
3922 git::git(
3925 &f.repo,
3926 &["update-ref", "refs/remotes/origin/main", &f.base],
3927 )
3928 .await
3929 .unwrap();
3930 let candidate = state.candidates[0].clone();
3931 if !missing {
3932 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3933 git::rescue_commit(&f.repo, "advance base before rebase")
3934 .await
3935 .unwrap();
3936 git::git(&candidate.worktree, &["rebase", "main"])
3937 .await
3938 .unwrap();
3939 git::sync_to_head(&candidate.worktree).await.unwrap();
3942 }
3943 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3944 state.reviewed_commits = Some(Vec::new());
3946 state.candidates[0].worktree = state.worktree_root().join("under-review");
3947 git::git(
3948 &f.repo,
3949 &[
3950 "worktree",
3951 "move",
3952 candidate.worktree.to_str().unwrap(),
3953 state.candidates[0].worktree.to_str().unwrap(),
3954 ],
3955 )
3956 .await
3957 .unwrap();
3958 let reviewer = state.worktree_root().join("review-1");
3959 git::worktree_add_detached(&f.repo, &reviewer, &head)
3960 .await
3961 .unwrap();
3962 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3965 git::rescue_commit(&reviewer, "old review history")
3966 .await
3967 .unwrap();
3968 git::git(&reviewer, &["reset", "--hard", &head])
3969 .await
3970 .unwrap();
3971 if missing {
3972 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3973 }
3974 let run_dir = f.home.join("runs").join(&state.id);
3975 let artifacts = run_dir.join("artifacts");
3976 std::fs::create_dir_all(&artifacts).unwrap();
3977 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3978 if missing {
3979 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3980 }
3981 state.status = RunStatus::Merged;
3982 state.merge = Some(crate::run::MergeOutcome {
3983 mode: crate::config::MergeMode::Pr,
3984 ok: true,
3985 detail: "confirmed squash merge".into(),
3986 empty: false,
3987 });
3988 state.save_under(&f.home).unwrap();
3989 let owned = state.candidates[0].worktree.clone();
3991 state.candidates[0].worktree = f.home.join("missing-external");
3992 assert!(
3993 crate::graph::fold_run(&mut state, true, &f.home)
3994 .await
3995 .unwrap_err()
3996 .to_string()
3997 .contains("outside the inventoried")
3998 );
3999 state.candidates[0].worktree = owned;
4000 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
4002 assert!(
4003 crate::graph::fold_run(&mut state, true, &f.home)
4004 .await
4005 .is_err()
4006 );
4007 assert!(reviewer.exists());
4008 drop(guard);
4009 let evidence = artifacts.join("fold-evidence");
4011 std::fs::write(&evidence, "blocked").unwrap();
4012 assert!(
4013 crate::graph::fold_run(&mut state, true, &f.home)
4014 .await
4015 .is_err()
4016 );
4017 assert!(reviewer.exists());
4018 std::fs::remove_file(&evidence).unwrap();
4019 std::fs::create_dir_all(&evidence).unwrap();
4020 let oversized = evidence.join("capacity-fixture");
4021 std::fs::File::create(&oversized)
4022 .unwrap()
4023 .set_len(64 * 1024 * 1024)
4024 .unwrap();
4025 assert!(
4026 crate::graph::fold_run(&mut state, true, &f.home)
4027 .await
4028 .is_err()
4029 );
4030 assert!(reviewer.exists());
4031 std::fs::remove_file(oversized).unwrap();
4032 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
4035 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
4036 assert!(
4037 crate::graph::fold_run(&mut state, true, &f.home)
4038 .await
4039 .is_err()
4040 );
4041 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
4042 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
4043 .await
4044 .unwrap();
4045 std::fs::write(&checkpoint, &head).unwrap();
4046 assert!(
4047 crate::graph::fold_run(&mut state, true, &f.home)
4048 .await
4049 .is_err()
4050 );
4051 std::fs::remove_file(checkpoint).unwrap();
4052 let (folded, unreadable) = crate::clean::fold_due(
4053 &f.home.join("runs"),
4054 &f.home,
4055 &f.bay,
4056 &Fx::disk(),
4057 Fx::now(),
4058 )
4059 .await
4060 .unwrap();
4061 if folded != 1 {
4062 panic!(
4063 "normal cleanup failed: {:#}",
4064 crate::graph::fold_run(&mut state, true, &f.home)
4065 .await
4066 .unwrap_err()
4067 );
4068 }
4069 assert_eq!((folded, unreadable), (1, 0));
4070 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4071 state = RunState::load_under(&state.id, &f.home).unwrap();
4072 assert!(state.candidates[0].folded);
4073 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4074 crate::graph::fold_run(&mut state, true, &f.home)
4075 .await
4076 .unwrap();
4077 assert_eq!(
4078 std::fs::read(evidence.join("manifest.json")).unwrap(),
4079 manifest
4080 );
4081 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4082 .await
4083 .unwrap();
4084 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4085 assert!(
4086 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4087 .await
4088 .unwrap()
4089 .ok()
4090 );
4091 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4094 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4095 .as_str()
4096 .unwrap();
4097 let recovered = f.home.join("recovered");
4098 std::fs::create_dir_all(&recovered).unwrap();
4099 git::git(&recovered, &["init"]).await.unwrap();
4100 std::fs::write(
4101 recovered.join("source"),
4102 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4103 )
4104 .unwrap();
4105 git::git(
4106 &recovered,
4107 &[
4108 "apply",
4109 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4110 ],
4111 )
4112 .await
4113 .unwrap();
4114 assert_eq!(
4115 std::fs::read_to_string(recovered.join("source")).unwrap(),
4116 "cc05"
4117 );
4118 assert!(
4119 !std::fs::read(evidence.join(format!("{head}.patch")))
4120 .unwrap()
4121 .is_empty()
4122 );
4123 assert_eq!(
4124 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4125 "review findings"
4126 );
4127 if missing {
4128 assert_eq!(
4129 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4130 "original snapshot"
4131 );
4132 } else {
4133 assert!(!artifacts.join("cand-A.patch").exists());
4134 }
4135 assert!(run_dir.join("run.json").exists());
4136 }
4137 }
4138
4139 #[tokio::test]
4140 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4141 use std::os::unix::fs::PermissionsExt as _;
4142 let f = Fx::new().await;
4143 let source = f.run("dd03", true).await;
4144 let empty = f.run("dd04", false).await;
4145 let task = f.task(&[source, empty.clone()]);
4146 let entered = f.repo.join("proof-entered");
4147 let release = f.repo.join("proof-release");
4148 let hook = f.repo.join("proof-monitor");
4149 std::fs::write(&hook, format!(
4150 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4151 entered.display(), release.display())).unwrap();
4152 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4153 git::git(
4154 &f.repo,
4155 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4156 )
4157 .await
4158 .unwrap();
4159 struct ReleaseProof(PathBuf);
4160 impl Drop for ReleaseProof {
4161 fn drop(&mut self) {
4162 let _ = std::fs::write(&self.0, "continue");
4163 }
4164 }
4165 let _release_on_panic = ReleaseProof(release.clone());
4166 let home = f.home.clone();
4167 let sweep = tokio::spawn(async move {
4168 let mut report = Report::default();
4169 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4170 report
4171 });
4172 let watchdog = std::time::Duration::from_secs(60);
4175 tokio::time::timeout(watchdog, async {
4176 while !entered.exists() {
4177 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4178 }
4179 })
4180 .await
4181 .expect("Git retirement proof did not enter the blocking hook");
4182 let queue = crate::queue::Queue::at(f.home.join("queue"));
4183 let mut selected = queue.get(&task.id).unwrap();
4184 selected.resume_override = Some(crate::queue::OperatorResume {
4185 question_id: "during-slow-proof".into(),
4186 at: Fx::now(),
4187 conductor_rehold: None,
4188 forced: false,
4189 pinned_run: Some(empty.id.clone()),
4190 });
4191 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4192 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4196 Ok(result) => {
4197 result.unwrap().unwrap();
4198 true
4199 }
4200 Err(_) => {
4201 std::fs::write(&release, "continue").unwrap();
4202 update.await.unwrap().unwrap();
4203 false
4204 }
4205 };
4206 std::fs::write(release, "continue").unwrap();
4207 let report = tokio::time::timeout(watchdog, sweep)
4208 .await
4209 .expect("Git retirement proof did not finish after release")
4210 .unwrap();
4211 assert!(
4212 updated_while_blocked,
4213 "task writers must not wait behind Git retirement proof"
4214 );
4215 assert_eq!(report.retired, 0);
4216 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4217 assert!(empty.candidates[0].worktree.exists());
4218 assert!(
4219 RunState::load_under(&empty.id, &f.home)
4220 .unwrap()
4221 .retired_to
4222 .is_none()
4223 );
4224 }
4225
4226 #[tokio::test]
4227 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4228 let f = Fx::new().await;
4229 let source = f.run("dd01", true).await;
4230 let mut empty = f.run("dd02", false).await;
4231 let expected = f.task(&[source.clone(), empty.clone()]);
4232 let queue = crate::queue::Queue::at(f.home.join("queue"));
4233 let _claim = queue.claim(&expected.id).unwrap();
4234 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4236 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4237 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4238 let mut selected = queue.get(&expected.id).unwrap();
4239 selected.resume_override = Some(crate::queue::OperatorResume {
4240 question_id: "new-owner-decision".into(),
4241 at: Fx::now(),
4242 conductor_rehold: None,
4243 forced: false,
4244 pinned_run: Some(empty.id.clone()),
4245 });
4246 queue.put(&mut selected).unwrap();
4247 let error = commit_retirement(
4248 &f.home,
4249 &queue,
4250 &expected,
4251 &mut empty,
4252 &source.id,
4253 &expected.runs,
4254 Fx::now(),
4255 )
4256 .unwrap_err();
4257 assert!(error.to_string().contains("task changed"));
4258 assert!(
4259 RunState::load_under(&empty.id, &f.home)
4260 .unwrap()
4261 .retired_to
4262 .is_none()
4263 );
4264 assert!(empty.candidates[0].worktree.exists());
4265 assert_eq!(
4266 queue
4267 .get(&expected.id)
4268 .unwrap()
4269 .resume_override
4270 .unwrap()
4271 .pinned_run,
4272 Some(empty.id.clone())
4273 );
4274 assert!(
4275 !f.home
4276 .join("recovery-lineages")
4277 .join(format!("{}.json", expected.id))
4278 .exists()
4279 );
4280 assert!(
4282 commit_retirement(
4283 &f.home,
4284 &queue,
4285 &selected,
4286 &mut empty,
4287 &source.id,
4288 &selected.runs,
4289 Fx::now()
4290 )
4291 .unwrap_err()
4292 .to_string()
4293 .contains("selected this recovery source")
4294 );
4295 }
4296
4297 #[tokio::test]
4298 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4299 let f = Fx::new().await;
4300 let state = f.run("cccc", false).await;
4301 let wt = &state.candidates[0].worktree;
4302 Fx::target(wt);
4303 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4304 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4305 assert!(
4306 !declared_outputs(&wt.join("target"))
4307 .unwrap()
4308 .contains(&wt.join("target/debug/link"))
4309 );
4310 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4311 let cargo = cargo_locks(&wt.join("target")).unwrap();
4312 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4313 assert_eq!(out.freed_bytes, 0);
4314 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4315 drop(cargo);
4316 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4317 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4318 assert_eq!(out.freed_bytes, 0);
4319 drop(driver);
4320 let exclusive = OutputLease::take(wt, true).unwrap();
4321 assert!(
4322 OutputLease::borrow(wt).is_err(),
4323 "resume cannot start a writer during cleanup"
4324 );
4325 drop(exclusive);
4326 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4327 assert_eq!(
4328 std::fs::read(wt.join("target/notes")).unwrap(),
4329 b"user file"
4330 );
4331 }
4332
4333 #[tokio::test]
4334 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4335 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4336 let f = Fx::new().await;
4337 let source = f.run("aa10", true).await;
4338 let empty = f.run("aa11", false).await;
4339 f.task(&[source, empty.clone()]);
4340 let wt = &empty.candidates[0].worktree;
4341 Fx::target(wt);
4342 let path = wt.join("target").join(leftover);
4343 if leftover.ends_with(".log") {
4344 std::fs::write(&path, "keep log\n").unwrap();
4345 } else {
4346 std::fs::create_dir(&path).unwrap();
4348 }
4349 let mut disk = Fx::disk();
4350 disk.auto_fold = true;
4351 let report = sweep(&f.home, &disk, Fx::now()).await;
4352 assert_eq!(report.freed_bytes, 200, "{report:?}");
4353 assert_eq!(report.retired, 0, "{report:?}");
4354 assert!(!wt.join("target/debug/output").exists());
4355 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4356 assert!(path.exists());
4357 if leftover.ends_with(".log") {
4358 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4359 }
4360 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4361 assert!(worktrees(&empty).await.unwrap().contains(wt));
4362 assert!(
4363 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4364 .await
4365 .unwrap()
4366 );
4367 assert!(
4368 RunState::load_under(&empty.id, &f.home)
4369 .unwrap()
4370 .retired_to
4371 .is_none()
4372 );
4373 }
4374 }
4375
4376 #[tokio::test]
4377 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4378 let f = Fx::new().await;
4379 let state = f.run("aa12", false).await;
4380 let wt = &state.candidates[0].worktree;
4381 Fx::target(wt);
4382 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4383 assert_eq!(report.freed_bytes, 200, "{report:?}");
4384 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4385 for name in ["tmp", "magi-tools", "log"] {
4386 let path = wt.join("target").join(name);
4387 std::fs::create_dir(&path).unwrap();
4388 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4389 std::fs::remove_dir(path).unwrap();
4390 }
4391 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4392 }
4393
4394 #[tokio::test]
4395 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4396 for invalid in [
4397 "tag",
4398 "identity",
4399 "tracked",
4400 "profile-file",
4401 "profile-symlink",
4402 "identity-symlink",
4403 ] {
4404 let f = Fx::new().await;
4405 let state = f.run("aa13", false).await;
4406 let wt = &state.candidates[0].worktree;
4407 Fx::target(wt);
4408 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4409 let target = wt.join("target");
4410 match invalid {
4411 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4412 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4413 "tracked" => {
4414 git::git(wt, &["add", "-f", "target/debug/output"])
4415 .await
4416 .unwrap();
4417 }
4418 "profile-file" => {
4419 std::fs::write(target.join("release"), "keep").unwrap();
4420 }
4421 "profile-symlink" => {
4422 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4423 .unwrap();
4424 }
4425 "identity-symlink" => {
4426 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4427 .unwrap();
4428 std::os::unix::fs::symlink(
4429 target.join("saved-info"),
4430 target.join(".rustc_info.json"),
4431 )
4432 .unwrap();
4433 }
4434 _ => unreachable!(),
4435 }
4436 let before = std::fs::read(target.join("debug/output")).unwrap();
4437 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4438 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4439 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4440 assert!(target.join("tmp").is_dir());
4441 }
4442 }
4443
4444 #[tokio::test]
4445 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4446 let f = Fx::new().await;
4447 let source = f.run("0000", true).await;
4448 let empty = f.run("0001", false).await;
4449 let changed = f.run("0002", false).await;
4450 let unknown = f.run("0003", false).await;
4451 let unproven = f.run("0004", false).await;
4452 f.task(&[
4453 source.clone(),
4454 empty.clone(),
4455 changed.clone(),
4456 unknown.clone(),
4457 unproven.clone(),
4458 ]);
4459 for run in [&empty, &changed, &unknown] {
4460 Fx::target(&run.candidates[0].worktree);
4461 }
4462 let reviewer = empty.worktree_root().join("review-1");
4463 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4464 .await
4465 .unwrap();
4466 Fx::target(&reviewer);
4467 let mut disk = Fx::disk();
4468 disk.auto_fold = false;
4469 let report = sweep(&f.home, &disk, Fx::now()).await;
4470 assert_eq!(report.freed_bytes, 800, "{report:?}");
4471 let wt = &empty.candidates[0].worktree;
4472 assert!(wt.join("target/debug/output.d").exists());
4473 assert!(!wt.join("target/debug/output").exists());
4474 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4475 let proof: CargoResidue =
4476 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4477 assert_eq!(
4478 proof.files[Path::new("debug/output.d")],
4479 std::fs::read(wt.join("target/debug/output.d")).unwrap()
4480 );
4481 std::fs::write(
4482 changed.candidates[0].worktree.join("target/debug/output.d"),
4483 "owner notes",
4484 )
4485 .unwrap();
4486 std::fs::write(
4487 unknown.candidates[0].worktree.join("target/debug/notes"),
4488 "keep",
4489 )
4490 .unwrap();
4491 Fx::target(&unproven.candidates[0].worktree);
4492 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4493 disk.auto_fold = true;
4494 let locks = cargo_locks(&wt.join("target")).unwrap();
4495 let mut report = Report::default();
4496 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4497 assert_eq!(report.retired, 0, "{report:?}");
4498 assert!(
4499 report
4500 .retained
4501 .iter()
4502 .any(|reason| reason.contains("Cargo build"))
4503 );
4504 drop(locks);
4505 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4506 assert_eq!(report.retired, 1, "{report:?}");
4507 assert!(!wt.exists());
4508 assert!(!reviewer.exists());
4509 assert!(
4510 residue_proof(&f.home, &empty.id, &reviewer)
4511 .unwrap()
4512 .exists()
4513 );
4514 assert!(
4515 proof_path.exists(),
4516 "metadata evidence outlives the worktree"
4517 );
4518 assert_eq!(
4519 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4520 Some(source.id)
4521 );
4522 for run in [&changed, &unknown, &unproven] {
4523 assert!(run.candidates[0].worktree.exists());
4524 assert!(
4525 RunState::load_under(&run.id, &f.home)
4526 .unwrap()
4527 .retired_to
4528 .is_none()
4529 );
4530 }
4531 }
4532
4533 #[tokio::test]
4534 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4535 let f = Fx::new().await;
4536 let source = f.run("0000", true).await;
4537 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4538 .await
4539 .unwrap();
4540 let mut runs = vec![source.clone()];
4541 for n in 1..=12 {
4542 runs.push(f.run(&format!("{n:04}"), false).await);
4543 }
4544 let dirty = runs[1].candidates[0].worktree.clone();
4545 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4546 let ignored = runs[2].candidates[0].worktree.clone();
4547 std::fs::create_dir(ignored.join("scratch")).unwrap();
4548 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4549 runs[1].candidates.clear();
4552 runs[1].save_under(&f.home).unwrap();
4553 let task = f.task(&runs);
4554 let mut total = 0;
4555 for _ in 0..4 {
4556 let mut report = Report::default();
4557 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4558 assert!(report.retired <= 5);
4559 total += report.retired;
4560 }
4561 assert_eq!(total, 10);
4562 assert!(dirty.exists());
4563 assert!(ignored.exists());
4564 assert_eq!(
4565 git::rev_parse(&f.repo, &source.candidates[0].branch)
4566 .await
4567 .unwrap(),
4568 source_head
4569 );
4570 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4571 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4572 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4573 let mut released = task.clone();
4574 released.release();
4575 assert_eq!(
4576 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4577 RunState::load_under(id, &f.home)
4578 }),
4579 Some(source.id.clone())
4580 );
4581
4582 assert!(
4583 f.home
4584 .join("runs")
4585 .join(&retired.id)
4586 .join("run.json")
4587 .exists()
4588 );
4589 assert!(
4590 f.home
4591 .join("recovery-lineages")
4592 .join(format!("{}.json", task.id))
4593 .exists()
4594 );
4595 }
4596
4597 #[tokio::test]
4598 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4599 let f = Fx::new().await;
4600 let a = f.run("dddd", true).await;
4601 let b = f.run("eeee", true).await;
4602 let empty = f.run("ffff", false).await;
4603 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4604 assert!(
4605 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4606 "unfinished queue history protects older source branches from ordinary folding"
4607 );
4608 let mut prior = a.clone();
4609 assert!(
4610 crate::graph::fold_run(&mut prior, true, &f.home)
4611 .await
4612 .unwrap_err()
4613 .to_string()
4614 .contains(&task.id)
4615 );
4616 let mut report = Report::default();
4617 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4618 assert_eq!(report.retired, 0);
4619 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4620 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4621 Fx::target(&a.candidates[0].worktree);
4622 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4623 assert!(
4624 report.freed_bytes > 0,
4625 "source retention does not retain regenerable outputs"
4626 );
4627 assert!(a.candidates[0].worktree.exists());
4628 assert!(b.candidates[0].worktree.exists());
4629 assert!(empty.candidates[0].worktree.exists());
4630 }
4631
4632 #[tokio::test]
4633 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4634 let f = Fx::new().await;
4635 let authority = f.run("p000", true).await;
4636 let question_run = f.run("p001", false).await;
4637 let mut checkpoint = f.run("p002", false).await;
4638 checkpoint.land_approval = Some(crate::run::LandApproval {
4639 question: "pending".into(),
4640 head: f.base.clone(),
4641 });
4642 checkpoint.save_under(&f.home).unwrap();
4643 let retained = f.run("p003", false).await;
4644 std::fs::write(
4645 f.home
4646 .join("runs")
4647 .join(&retained.id)
4648 .join("retain-recovery-sources"),
4649 "owner",
4650 )
4651 .unwrap();
4652 let mut partial = f.run("p004", false).await;
4653 let reviewer = partial.worktree_root().join("review-1");
4654 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4655 .await
4656 .unwrap();
4657 partial.retired_to = Some(authority.id.clone());
4659 partial.save_under(&f.home).unwrap();
4660 assert!(
4661 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4662 .await
4663 .unwrap()
4664 );
4665 let git_checkpoint = f.run("p005", false).await;
4666 let location = git::git(
4667 &git_checkpoint.candidates[0].worktree,
4668 &[
4669 "rev-parse",
4670 "--path-format=absolute",
4671 "--git-path",
4672 "sequencer",
4673 ],
4674 )
4675 .await
4676 .unwrap();
4677 std::fs::create_dir(location.trim()).unwrap();
4678 let mut preservation = f.run("p006", false).await;
4679 preservation
4680 .candidate_preservation
4681 .insert('A', "pending preservation".into());
4682 preservation.save_under(&f.home).unwrap();
4683 let task = f.task(&[
4684 preservation.clone(),
4685 git_checkpoint.clone(),
4686 authority.clone(),
4687 question_run.clone(),
4688 checkpoint.clone(),
4689 retained.clone(),
4690 partial.clone(),
4691 ]);
4692 let questions = crate::ask::Questions::at(f.home.join("questions"));
4693 let mut question = crate::ask::Question::new(
4694 question_run.id.clone(),
4695 "implement".into(),
4696 "fixture".into(),
4697 "owner decision".into(),
4698 String::new(),
4699 vec!["keep".into()],
4700 );
4701 questions.put(&mut question).unwrap();
4702 let mut report = Report::default();
4703 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4704 assert_eq!(report.retired, 1, "{report:?}");
4705 assert!(!reviewer.exists());
4706 for state in [
4707 &question_run,
4708 &checkpoint,
4709 &retained,
4710 &git_checkpoint,
4711 &preservation,
4712 ] {
4713 assert!(state.candidates[0].worktree.exists());
4714 assert!(
4715 RunState::load_under(&state.id, &f.home)
4716 .unwrap()
4717 .retired_to
4718 .is_none()
4719 );
4720 }
4721 assert!(questions.get(&question.id).unwrap().status.open());
4722 assert_eq!(
4723 crate::queue::Queue::at(f.home.join("queue"))
4724 .get(&task.id)
4725 .unwrap()
4726 .status,
4727 task.status
4728 );
4729 }
4730
4731 #[tokio::test]
4732 async fn preservation_schema_migration_remains_distinct_from_retirement() {
4733 let f = Fx::new().await;
4734 let mut state = f.run("p007", false).await;
4735 state.schema = 15;
4736 state.save_under(&f.home).unwrap();
4737 let native = RunState::load_under(&state.id, &f.home).unwrap();
4738 assert_eq!(native.schema, crate::run::SCHEMA);
4739 assert!(native.candidate_preservation.is_empty());
4740 assert!(native.retired_to.is_none());
4741 state.schema = 14;
4742 state.save_under(&f.home).unwrap();
4743 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4744 assert!(legacy.candidate_preservation.contains_key(&'A'));
4745 legacy.save_under(&f.home).unwrap();
4746 let restored = RunState::load_under(&state.id, &f.home).unwrap();
4747 assert_eq!(
4748 restored.candidate_preservation,
4749 legacy.candidate_preservation
4750 );
4751 }
4752
4753 #[tokio::test]
4754 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4755 let f = Fx::new().await;
4756 let cache = f.repo.join("cache");
4757 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4758 std::fs::create_dir_all(&cache).unwrap();
4759 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4760 assert!(
4761 crate::cache::maintenance_prune(&f.home, &cache, 1)
4762 .unwrap()
4763 .is_none()
4764 );
4765 drop(guard);
4766 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4767 }
4768
4769 #[tokio::test]
4770 async fn launcher_death_does_not_release_live_tool_output_reservations() {
4771 let f = Fx::new().await;
4772 let run = f.run("abcd", false).await;
4773 let wt = &run.candidates[0].worktree;
4774 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4775 helper
4776 .quiet()
4777 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4778 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4779 .env("MAGI_TEST_WORKTREE", wt)
4780 .stdin(std::process::Stdio::null())
4781 .stdout(std::process::Stdio::null())
4782 .stderr(std::process::Stdio::null())
4783 .kill_on_drop(true);
4784 assert!(
4785 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4786 .await
4787 .unwrap()
4788 .unwrap()
4789 .success()
4790 );
4791 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4792 .unwrap()
4793 .trim()
4794 .parse()
4795 .unwrap();
4796 struct KillTool(i32);
4797 impl Drop for KillTool {
4798 fn drop(&mut self) {
4799 unsafe { libc::kill(self.0, libc::SIGKILL) };
4801 }
4802 }
4803 let tool = KillTool(pid);
4804 assert!(OutputLease::take(wt, true).is_err());
4805 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4806 drop(tool);
4807 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4808 loop {
4809 if OutputLease::take(wt, true).is_ok()
4810 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4811 {
4812 break;
4813 }
4814 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4815 }
4816 })
4817 .await
4818 .unwrap();
4819 }
4820
4821 #[tokio::test]
4822 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4823 let f = Fx::new().await;
4824 let run = f.run("abcd", false).await;
4825 let wt = &run.candidates[0].worktree;
4826 let cache = f.repo.join("cache");
4827 std::fs::create_dir(&cache).unwrap();
4828 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4829 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4830 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4831 cmd.quiet()
4832 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4833 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4834 .stdin(std::process::Stdio::null())
4835 .stdout(std::process::Stdio::null())
4836 .stderr(std::process::Stdio::null())
4837 .kill_on_drop(true);
4838 if clear_owner {
4839 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4840 } else {
4841 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4842 }
4843 let gib = 1024 * 1024 * 1024;
4844 let mut leases = vec![
4845 OutputLease::borrow(wt).unwrap().unwrap(),
4846 OutputLease::cache(&f.home, &cache, false).unwrap(),
4847 ];
4848 leases.extend(
4849 OutputLease::admit_with(
4850 &f.home,
4851 std::slice::from_ref(wt),
4852 8 * gib,
4853 |_| Ok(9 * gib),
4854 )
4855 .unwrap(),
4856 );
4857 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4858 assert!(
4859 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4860 .await
4861 .unwrap()
4862 .unwrap()
4863 .success()
4864 );
4865 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4866 struct KillTool(Option<i32>);
4867 impl Drop for KillTool {
4868 fn drop(&mut self) {
4869 if let Some(pid) = self.0 {
4871 unsafe { libc::kill(pid, libc::SIGKILL) };
4872 }
4873 }
4874 }
4875 let mut tool = KillTool(Some(pid));
4876 assert!(OutputLease::take(wt, true).is_err());
4880 let finish = if cancel {
4881 drop(running);
4882 None
4883 } else {
4884 Some(tokio::spawn(async move { running.finish().await }))
4885 };
4886 if clear_owner {
4887 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4888 assert!(crate::proc::pid_alive(pid as u32));
4889 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4890 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4891 assert!(
4892 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4893 9 * gib
4894 ))
4895 .is_err(),
4896 "orphan still owns its disk reservation"
4897 );
4898 assert!(
4899 crate::cache::maintenance_prune(&f.home, &cache, 0)
4900 .unwrap()
4901 .is_none()
4902 );
4903 if let Some(finish) = &finish {
4904 assert!(!finish.is_finished());
4905 }
4906 unsafe { libc::kill(pid, libc::SIGKILL) };
4908 }
4909 if let Some(finish) = finish {
4910 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4911 .await
4912 .unwrap()
4913 .unwrap();
4914 }
4915 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4916 loop {
4917 if OutputLease::take(wt, true).is_ok()
4918 && OutputLease::cache(&f.home, &cache, true).is_ok()
4919 && OutputLease::admit_with(
4920 &f.home,
4921 std::slice::from_ref(wt),
4922 8 * gib,
4923 |_| Ok(9 * gib),
4924 )
4925 .is_ok()
4926 {
4927 if !clear_owner {
4929 assert!(!crate::proc::pid_alive(pid as u32));
4930 }
4931 break;
4932 }
4933 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4934 }
4935 })
4936 .await
4937 .unwrap();
4938 tool.0 = None;
4939 }
4940 }
4941
4942 #[tokio::test]
4943 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4944 let f = Fx::new().await;
4945 let run = f.run("abcd", false).await;
4946 let wt = &run.candidates[0].worktree;
4947 let cache = f.repo.join("cache");
4948 std::fs::create_dir(&cache).unwrap();
4949 let pid_file = f.repo.join("child-pid");
4950 let mut cmd = tokio::process::Command::new("sh");
4951 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4952 .quiet()
4953 .env("CHILD_PID_FILE", &pid_file)
4954 .process_group(0)
4955 .kill_on_drop(true);
4956 let running = crate::seat_process::Running::spawn(
4957 cmd,
4958 None,
4959 vec![
4960 OutputLease::borrow(wt).unwrap().unwrap(),
4961 OutputLease::cache(&f.home, &cache, false).unwrap(),
4962 ],
4963 )
4964 .unwrap();
4965 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4966 while !pid_file.exists() {
4967 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4968 }
4969 })
4970 .await
4971 .unwrap();
4972 assert!(OutputLease::take(wt, true).is_err());
4973 assert!(
4974 crate::cache::maintenance_prune(&f.home, &cache, 0)
4975 .unwrap()
4976 .is_none()
4977 );
4978 let pid: u32 = std::fs::read_to_string(pid_file)
4979 .unwrap()
4980 .trim()
4981 .parse()
4982 .unwrap();
4983 drop(running);
4984 tokio::time::timeout(std::time::Duration::from_secs(10), async {
4985 loop {
4986 if let Ok(guard) = OutputLease::take(wt, true) {
4987 assert!(
4988 !crate::proc::pid_alive(pid),
4989 "output lock released before tool death"
4990 );
4991 drop(guard);
4992 break;
4993 }
4994 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4995 }
4996 })
4997 .await
4998 .unwrap();
4999 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5000 }
5001
5002 #[tokio::test]
5003 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
5004 let f = Fx::new().await;
5005 let reason = tokio::time::timeout(
5006 std::time::Duration::from_secs(2),
5007 emergency(vec![f.repo.clone()], u64::MAX),
5008 )
5009 .await
5010 .unwrap();
5011 assert!(reason.contains("disk pressure"));
5012 assert!(
5013 tokio::time::timeout(
5014 std::time::Duration::from_millis(10),
5015 emergency(vec![f.repo.clone()], 0)
5016 )
5017 .await
5018 .is_err()
5019 );
5020 assert_eq!(
5021 std::fs::read_to_string(f.repo.join("source")).unwrap(),
5022 "base\n"
5023 );
5024 }
5025
5026 #[tokio::test]
5027 async fn automatic_requeues_cannot_refund_failed_input_generation() {
5028 let f = Fx::new().await;
5029 let mut run = f.run("1234", false).await;
5030 run.prep_failure = Some("cherry-pick conflict".into());
5031 let mut task = f.task(std::slice::from_ref(&run));
5032 let inputs = crate::queue::recovery_inputs(&run.sources);
5033 for _ in 0..108 {
5034 task.recovery_failed(&run);
5035 task.requeue();
5036 assert_eq!(task.attempts, 0);
5037 }
5038 let bytes = serde_json::to_vec(&task).unwrap();
5039 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
5040 assert!(
5041 restored
5042 .recovery_exhausted(&run.base_commit, &inputs, 2)
5043 .is_some()
5044 );
5045 assert!(
5046 restored
5047 .recovery_exhausted("different-base", &inputs, 2)
5048 .is_none()
5049 );
5050 let original_base = run.base_commit.clone();
5051 run.base_commit = "different-base".into();
5052 restored.recovery_failed(&run);
5053 assert!(
5054 restored
5055 .recovery_exhausted(&original_base, &inputs, 2)
5056 .is_some(),
5057 "returning to old inputs does not replenish their budget"
5058 );
5059 assert!(
5060 restored
5061 .recovery_exhausted(&run.base_commit, &inputs, 2)
5062 .is_none()
5063 );
5064 restored.release_operator("test-owner", None);
5065 assert!(
5066 restored
5067 .recovery_exhausted(&run.base_commit, &inputs, 2)
5068 .is_none()
5069 );
5070 }
5071 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5072 let mut state = f.run(tag, useful).await;
5073 state.status = RunStatus::Superseded;
5074 state.save_under(&f.home).unwrap();
5075 state
5076 }
5077
5078 fn historical_review(state: &mut RunState, head: &str) {
5079 state.reviews.push(
5080 serde_json::from_value(serde_json::json!({
5081 "round": state.reviews.len() + 1, "head": head, "reviews": []
5082 }))
5083 .unwrap(),
5084 );
5085 }
5086
5087 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5088 let path = state.worktree_root().join("review-1");
5089 git::worktree_add_detached(&f.repo, &path, head)
5090 .await
5091 .unwrap();
5092 path
5093 }
5094
5095 #[tokio::test]
5096 async fn source_eviction_separates_run_authority_from_read_seats() {
5097 for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5098 let f = Fx::new().await;
5099 let mut state = inactive_source(&f, "ef01", true).await;
5100 state.status = status;
5101 state.pr = Some(
5102 serde_json::from_value(serde_json::json!({
5103 "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5104 "checks":"green", "round":1, "rounds":1
5105 }))
5106 .unwrap(),
5107 );
5108 state.pr_head = Some(state.candidates[0].branch.clone());
5109 state.pr_merged_head = Some(f.base.clone());
5110 state.recovery = Some(
5111 serde_json::from_value(serde_json::json!({
5112 "from_run": "20260101-000000-abcd", "generation": 1,
5113 "mode": "full-review"
5114 }))
5115 .unwrap(),
5116 );
5117 state.pr_recovery = Some(
5118 serde_json::from_value(serde_json::json!({
5119 "url": "https://example.invalid/pull/1", "branch": "public",
5120 "head": f.base, "source_run": state.id,
5121 "instruction": "keep implementation", "approved_at": Fx::now()
5122 }))
5123 .unwrap(),
5124 );
5125 state.save_under(&f.home).unwrap();
5126 let review = reviewer(&f, &state, &f.base).await;
5127 let mut other = inactive_source(&f, "ef02", true).await;
5128 other.pr_recovery = state.pr_recovery.clone();
5129 other.save_under(&f.home).unwrap();
5130 let mut task = f.task(&[state.clone(), other]);
5131 task.pr_recovery = state.pr_recovery.clone();
5132 crate::queue::Queue::at(f.home.join("queue"))
5133 .put(&mut task)
5134 .unwrap();
5135 let run_path = f.home.join("runs").join(&state.id).join("run.json");
5136 let before = std::fs::read(&run_path).unwrap();
5137 let task_before = serde_json::to_value(&task).unwrap();
5138 let mut report = Report::default();
5139 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5140 assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5141 assert!(!review.exists());
5142 assert!(state.candidates[0].worktree.exists());
5143 assert!(
5144 git::branch_exists(&f.repo, &state.candidates[0].branch)
5145 .await
5146 .unwrap()
5147 );
5148 assert_eq!(std::fs::read(&run_path).unwrap(), before);
5149 assert_eq!(
5150 serde_json::to_value(
5151 crate::queue::Queue::at(f.home.join("queue"))
5152 .get(&task.id)
5153 .unwrap()
5154 )
5155 .unwrap(),
5156 task_before
5157 );
5158 }
5159 }
5160
5161 #[tokio::test]
5162 async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5163 for kind in ["own-path", "active-seat", "paused-session"] {
5164 let f = Fx::new().await;
5165 let mut state = inactive_source(&f, "ef03", true).await;
5166 let review = reviewer(&f, &state, &f.base).await;
5167 let expected = match kind {
5168 "own-path" => {
5169 state.attachments.push(review.join("source"));
5170 "run.attachments[0]"
5171 }
5172 "active-seat" => {
5173 state.active.insert(
5174 "review-1".into(),
5175 serde_json::from_value(serde_json::json!({
5176 "node":"review", "started_at":Fx::now(), "timeout_secs":60
5177 }))
5178 .unwrap(),
5179 );
5180 "continuation lease"
5181 }
5182 _ => {
5183 state.status = RunStatus::Blocked;
5184 state.parked = true;
5185 let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5186 seat.turns = 2;
5187 seat.captured_session = Some("saved-session".into());
5188 state.seats.insert("review-1".into(), seat);
5189 "resume contract"
5190 }
5191 };
5192 state.save_under(&f.home).unwrap();
5193 let mut report = Report::default();
5194 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5195 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5196 assert!(review.exists());
5197 assert!(
5198 report.retained.iter().any(|r| r.contains(expected)),
5199 "{kind}: {report:?}"
5200 );
5201 restore_execution_seats(&f.home, &state).await.unwrap();
5204 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5205 }
5206 }
5207
5208 #[tokio::test]
5209 async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5210 for kind in ["waiting", "lease", "historical", "unknown"] {
5211 let f = Fx::new().await;
5212 let state = inactive_source(&f, "ef06", true).await;
5213 let review = reviewer(&f, &state, &f.base).await;
5214 let store = crate::ask::Questions::at(f.home.join("questions"));
5215 let mut question = crate::ask::Question::new(
5216 "20260101-000000-ffff".into(),
5217 "review".into(),
5218 "review-1".into(),
5219 "fixture".into(),
5220 "fixture".into(),
5221 Vec::new(),
5222 );
5223 question.cwd = Some(review.to_string_lossy().into_owned());
5224 if kind != "waiting" {
5225 question.status = crate::ask::QuestionStatus::Abandoned;
5226 }
5227 store.put(&mut question).unwrap();
5228 if kind == "lease" {
5229 store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5230 }
5231 if kind == "unknown" {
5232 let path = store.path_of(&question.id);
5233 let mut raw: serde_json::Value =
5234 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5235 raw["future_cwd_authority"] = serde_json::Value::Null;
5236 std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5237 }
5238 let mut report = Report::default();
5239 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5240 assert_eq!(
5241 report.evicted_copies,
5242 usize::from(kind == "historical"),
5243 "{kind}: {report:?}"
5244 );
5245 assert_eq!(review.exists(), kind != "historical");
5246 if kind == "waiting" || kind == "lease" {
5247 assert!(
5248 report
5249 .retained
5250 .iter()
5251 .any(|r| r.contains("question") && r.contains("cwd")),
5252 "{report:?}"
5253 );
5254 }
5255 }
5256 }
5257
5258 #[tokio::test]
5259 async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5260 let f = Fx::new().await;
5261 let mut state = inactive_source(&f, "ef07", true).await;
5262 let review = reviewer(&f, &state, &f.base).await;
5263 let mut report = Report::default();
5264 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5265 assert_eq!(report.evicted_copies, 1, "{report:?}");
5266 state.config.worktree.setup = vec![crate::config::SetupStep {
5267 run: Some("printf preserved > source; exit 1".into()),
5268 ..Default::default()
5269 }];
5270 state.save_under(&f.home).unwrap();
5271 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5272 let marker = f
5273 .home
5274 .join("runs")
5275 .join(&state.id)
5276 .join("artifacts/restore-review-1.pending");
5277 assert!(marker.exists());
5278 assert_eq!(
5279 std::fs::read_to_string(review.join("source")).unwrap(),
5280 "preserved"
5281 );
5282 assert!(
5283 restore_execution_seats(&f.home, &state)
5284 .await
5285 .unwrap_err()
5286 .to_string()
5287 .contains("interrupted execution restoration")
5288 );
5289 let mut retry = Report::default();
5290 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5291 assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5292 assert!(
5293 retry
5294 .retained
5295 .iter()
5296 .any(|r| r.contains("restoration checkpoint")),
5297 "{retry:?}"
5298 );
5299 assert!(marker.exists());
5300 assert_eq!(
5301 std::fs::read_to_string(review.join("source")).unwrap(),
5302 "preserved"
5303 );
5304 }
5305
5306 #[tokio::test]
5307 async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5308 for kind in [
5309 "alternates",
5310 "promisor-config",
5311 "promisor-pack",
5312 "objects-symlink",
5313 ] {
5314 let f = Fx::new().await;
5315 let state = inactive_source(&f, "ef11", true).await;
5316 let review = reviewer(&f, &state, &f.base).await;
5317 let common = crate::repo_lock::common_dir(&f.repo)
5318 .await
5319 .unwrap()
5320 .unwrap();
5321 match kind {
5322 "alternates" => {
5323 let borrowed = f._temp.path().join("borrowed-objects");
5324 std::fs::create_dir(&borrowed).unwrap();
5325 std::fs::write(
5326 common.join("objects/info/alternates"),
5327 format!("{}\n", borrowed.display()),
5328 )
5329 .unwrap();
5330 }
5331 "promisor-config" => {
5332 git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5333 .await
5334 .unwrap();
5335 }
5336 "promisor-pack" => {
5337 std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5338 }
5339 _ => {
5340 let borrowed = f._temp.path().join("borrowed-objects");
5341 std::fs::rename(common.join("objects"), &borrowed).unwrap();
5342 std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5343 }
5344 }
5345 let mut report = Report::default();
5346 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5347 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5348 assert!(review.exists());
5349 assert!(
5350 report.retained.iter().any(|r| r.contains("source anchor")),
5351 "{kind}: {report:?}"
5352 );
5353 }
5354 }
5355
5356 #[tokio::test]
5357 async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5358 let mut f = Fx::new().await;
5359 crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5360 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5361 assert_eq!(
5362 git::git(&f.repo, &["rev-list", "--count", &f.base])
5363 .await
5364 .unwrap(),
5365 "5001"
5366 );
5367 let state = inactive_source(&f, "ef10", true).await;
5368 let review = reviewer(&f, &state, &f.base).await;
5369 std::fs::write(review.join("source"), "bounded review delta").unwrap();
5370 git::git(&review, &["add", "."]).await.unwrap();
5371 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5372 let head = git::git(
5373 &review,
5374 &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5375 )
5376 .await
5377 .unwrap();
5378 git::git(&review, &["checkout", "--detach", &head])
5379 .await
5380 .unwrap();
5381 let run_file = f.home.join("runs").join(&state.id).join("run.json");
5382 let run_before = std::fs::read(&run_file).unwrap();
5383 let mut report = Report::default();
5384 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5385 assert_eq!(report.evicted_copies, 1, "{report:?}");
5386 assert!(!review.exists());
5387 assert!(state.candidates[0].worktree.exists());
5388 assert!(
5389 git::branch_exists(&f.repo, &state.candidates[0].branch)
5390 .await
5391 .unwrap()
5392 );
5393 assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5394 let archive = f
5395 .home
5396 .join("runs")
5397 .join(&state.id)
5398 .join("artifacts/source-evidence")
5399 .join(format!("{head}-{}", f.base));
5400 let manifest: serde_json::Value =
5401 serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5402 assert_eq!(manifest["anchor"], f.base);
5403 assert!(
5404 std::fs::metadata(archive.join("objects.pack"))
5405 .unwrap()
5406 .len()
5407 < 16 * 1024
5408 );
5409 let pin = format!("refs/magi/source-anchors/{}", f.base);
5410 assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5411 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5412 .await
5413 .unwrap();
5414 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5415 assert!(
5416 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5417 .await
5418 .unwrap()
5419 .ok()
5420 );
5421 let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5422 .await
5423 .unwrap();
5424 for kind in ["missing", "moved", "symbolic", "symlink"] {
5426 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5427 .await
5428 .unwrap();
5429 match kind {
5430 "moved" => {
5431 git::git(&f.repo, &["update-ref", &pin, &other])
5432 .await
5433 .unwrap();
5434 }
5435 "symbolic" => {
5436 git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5437 .await
5438 .unwrap();
5439 }
5440 "symlink" => {
5441 let common = crate::repo_lock::common_dir(&f.repo)
5442 .await
5443 .unwrap()
5444 .unwrap();
5445 let target = f._temp.path().join("operator-anchor");
5446 std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5447 let file = common.join(&pin);
5448 std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5449 std::os::unix::fs::symlink(target, file).unwrap();
5450 }
5451 _ => {}
5452 }
5453 assert!(
5454 restore_execution_seats(&f.home, &state).await.is_err(),
5455 "{kind}"
5456 );
5457 assert!(!review.exists());
5458 assert!(
5459 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5460 .await
5461 .unwrap()
5462 .ok()
5463 );
5464 if kind == "symlink" {
5465 let common = crate::repo_lock::common_dir(&f.repo)
5466 .await
5467 .unwrap()
5468 .unwrap();
5469 std::fs::remove_file(common.join(&pin)).unwrap();
5470 } else {
5471 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5472 .await
5473 .unwrap();
5474 }
5475 git::git(&f.repo, &["update-ref", &pin, &f.base])
5476 .await
5477 .unwrap();
5478 }
5479 assert!(
5482 git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5483 .await
5484 .unwrap()
5485 );
5486 git::git(
5487 &f.repo,
5488 &[
5489 "update-ref",
5490 "-d",
5491 &format!("refs/heads/{}", state.candidates[0].branch),
5492 ],
5493 )
5494 .await
5495 .unwrap();
5496 git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5497 .await
5498 .unwrap();
5499 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5500 .await
5501 .unwrap();
5502 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5503 assert_eq!(
5504 git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5505 .await
5506 .unwrap(),
5507 pin
5508 );
5509 assert_eq!(
5510 git::git(&f.repo, &["rev-list", "--count", &f.base])
5511 .await
5512 .unwrap(),
5513 "5001"
5514 );
5515 restore_execution_seats(&f.home, &state).await.unwrap();
5516 assert_eq!(
5517 git::git(&review, &["rev-list", "--count", "HEAD"])
5518 .await
5519 .unwrap(),
5520 "5002"
5521 );
5522 assert_eq!(
5523 git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5524 .await
5525 .unwrap(),
5526 "false"
5527 );
5528 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5529 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5530 let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5531 assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5532 std::fs::remove_dir_all(&f.repo).unwrap();
5533 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5534 let inspection = f._temp.path().join("shared-history-inspection");
5535 restore_seat(&f.home, &state.id, &head, &inspection)
5536 .await
5537 .unwrap();
5538 assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5539 assert_eq!(
5540 git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5541 .await
5542 .unwrap(),
5543 "true"
5544 );
5545 assert_eq!(
5546 git::git(&inspection, &["log", "--format=%H"])
5547 .await
5548 .unwrap(),
5549 format!("{head}\n{}", f.base)
5550 );
5551 assert!(!inspection.join(".git/objects/info/alternates").exists());
5552 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5553 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5554 }
5555
5556 #[tokio::test]
5557 async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5558 let f = Fx::new().await;
5559 let state = inactive_source(&f, "ef08", true).await;
5560 let review = reviewer(&f, &state, &f.base).await;
5561 let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5562 let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5563 .await
5564 .unwrap();
5565 let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
5566 .await
5567 .unwrap();
5568 git::git(&review, &["checkout", "--detach", &head])
5569 .await
5570 .unwrap();
5571 let archive =
5572 crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
5573 let mut report = Report::default();
5574 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5575 assert_eq!(report.evicted_copies, 0, "{report:?}");
5576 assert!(review.exists());
5577 assert!(
5578 report
5579 .retained
5580 .iter()
5581 .any(|r| r.contains("incomplete archived commit ancestry")),
5582 "{report:?}"
5583 );
5584 assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
5586 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5587 .await
5588 .unwrap();
5589 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5590 for oid in [&parent, &head] {
5591 assert!(
5592 !git::git_raw(&f.repo, &["cat-file", "-e", oid])
5593 .await
5594 .unwrap()
5595 .ok()
5596 );
5597 }
5598 let patch = source_diff(&archive).await.unwrap();
5599 assert!(
5600 crate::fold_evidence::import_snapshot(&archive, &f.repo)
5601 .await
5602 .is_err()
5603 );
5604 assert!(
5605 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5606 .await
5607 .unwrap()
5608 .ok()
5609 );
5610 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5611 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5612 let inspection = f._temp.path().join("legacy-tree-inspection");
5613 restore_source(&archive, &inspection).await.unwrap();
5614 assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
5615 assert!(
5616 !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
5617 .await
5618 .unwrap()
5619 .ok()
5620 );
5621 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5622 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5623 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5624 assert!(state.candidates[0].worktree.exists());
5625 }
5626
5627 #[tokio::test]
5628 async fn source_eviction_retains_ancestry_above_expanded_budget() {
5629 let f = Fx::new().await;
5630 let state = inactive_source(&f, "ef09", true).await;
5631 let review = reviewer(&f, &state, &f.base).await;
5632 std::fs::File::create(review.join("ancestor-only"))
5633 .unwrap()
5634 .set_len(64 * 1024 * 1024 + 1)
5635 .unwrap();
5636 git::git(&review, &["add", "."]).await.unwrap();
5637 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5638 let parent = git::git(
5639 &review,
5640 &["commit-tree", &parent_tree, "-m", "large ancestor"],
5641 )
5642 .await
5643 .unwrap();
5644 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5645 git::git(&review, &["add", "."]).await.unwrap();
5646 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5647 let head = git::git(
5648 &review,
5649 &[
5650 "commit-tree",
5651 &tree,
5652 "-p",
5653 &parent,
5654 "-m",
5655 "small final tree",
5656 ],
5657 )
5658 .await
5659 .unwrap();
5660 git::git(&review, &["checkout", "--detach", &head])
5661 .await
5662 .unwrap();
5663 let mut report = Report::default();
5664 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5665 assert_eq!(report.evicted_copies, 0, "{report:?}");
5666 assert!(
5667 report
5668 .retained
5669 .iter()
5670 .any(|r| r.contains("expanded object limit exceeded")),
5671 "{report:?}"
5672 );
5673 assert!(review.exists());
5674 assert!(state.candidates[0].worktree.exists());
5675 }
5676
5677 #[tokio::test]
5678 async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
5679 let f = Fx::new().await;
5680 let state = inactive_source(&f, "ef04", true).await;
5681 let review = reviewer(&f, &state, &f.base).await;
5682 std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
5683 git::git(&review, &["add", "."]).await.unwrap();
5684 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5685 let parent = git::git(
5686 &review,
5687 &["commit-tree", &parent_tree, "-m", "unrelated parent"],
5688 )
5689 .await
5690 .unwrap();
5691 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5692 std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
5693 git::git(&review, &["add", "."]).await.unwrap();
5694 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5695 let head = git::git(
5696 &review,
5697 &[
5698 "commit-tree",
5699 &tree,
5700 "-p",
5701 &parent,
5702 "-m",
5703 "unrelated review checkpoint",
5704 ],
5705 )
5706 .await
5707 .unwrap();
5708 git::git(&review, &["checkout", "--detach", &head])
5709 .await
5710 .unwrap();
5711 let mut report = Report::default();
5712 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5713 assert_eq!(report.evicted_copies, 1, "{report:?}");
5714 let ledger = f
5715 .home
5716 .join("runs")
5717 .join(&state.id)
5718 .join("artifacts/eviction-review-1.json");
5719 let original = std::fs::read(&ledger).unwrap();
5720 let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
5721 assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
5722 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5723 .await
5724 .unwrap();
5725 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5726 assert!(
5727 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5728 .await
5729 .unwrap()
5730 .ok()
5731 );
5732 assert!(
5733 !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
5734 .await
5735 .unwrap()
5736 .ok()
5737 );
5738 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5739 assert!(
5740 seat_heads(&f.home, &state.id)
5741 .await
5742 .unwrap()
5743 .contains_key(&head)
5744 );
5745 restore_execution_seats(&f.home, &state).await.unwrap();
5746 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
5747 assert_eq!(
5748 git::git(&review, &["log", "--format=%H"]).await.unwrap(),
5749 format!("{head}\n{parent}")
5750 );
5751 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5752 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5753 let inspection = f._temp.path().join("complete-history-inspection");
5754 restore_seat(&f.home, &state.id, &head, &inspection)
5755 .await
5756 .unwrap();
5757 assert_eq!(
5758 git::git(&inspection, &["log", "--format=%H"])
5759 .await
5760 .unwrap(),
5761 format!("{head}\n{parent}")
5762 );
5763 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5764 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5765
5766 assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
5767 assert!(
5768 !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
5769 .await
5770 .unwrap()
5771 .ok()
5772 );
5773 assert_eq!(
5774 std::fs::read_to_string(review.join("source")).unwrap(),
5775 "unrelated review checkpoint"
5776 );
5777 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5778 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5779 assert_eq!(
5780 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5781 "keep"
5782 );
5783 std::fs::remove_file(review.join("operator-notes")).unwrap();
5784 restore_execution_seats(&f.home, &state).await.unwrap();
5785 let mut second = Report::default();
5786 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
5787 assert_eq!(second.evicted_copies, 1, "{second:?}");
5788 assert_eq!(std::fs::read(&ledger).unwrap(), original);
5789 assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
5790 let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
5793 legacy["version"] = 1.into();
5794 legacy.as_object_mut().unwrap().remove("base");
5795 std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
5796 let mut advanced = state.clone();
5797 advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5798 .await
5799 .unwrap();
5800 advanced.save_under(&f.home).unwrap();
5801 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5802 let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
5804 let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
5805 let patch = std::fs::read(archive.join("change.patch")).unwrap();
5806 std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
5807 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5808 assert!(!review.exists());
5809 std::fs::write(archive.join("change.patch"), patch).unwrap();
5810 std::fs::create_dir(&review).unwrap();
5811 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5812 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5813 assert_eq!(
5814 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5815 "keep"
5816 );
5817 }
5818
5819 #[tokio::test]
5820 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
5821 use std::os::unix::fs::PermissionsExt;
5822 let mut f = Fx::new().await;
5823 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
5824 git::git(&f.repo, &["add", "."]).await.unwrap();
5825 git::git(&f.repo, &["commit", "-m", "binary base"])
5826 .await
5827 .unwrap();
5828 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5829 let mut state = inactive_source(&f, "ec01", true).await;
5830 let candidate = state.candidates[0].worktree.clone();
5831 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5832 historical_review(&mut state, &earlier);
5833 let review = reviewer(&f, &state, &earlier).await;
5834 std::fs::remove_file(candidate.join("source")).unwrap();
5835 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5836 std::fs::write(candidate.join("script"), "run\n").unwrap();
5837 std::fs::set_permissions(
5838 candidate.join("script"),
5839 std::fs::Permissions::from_mode(0o755),
5840 )
5841 .unwrap();
5842 git::git(&candidate, &["add", "."]).await.unwrap();
5843 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5844 .await
5845 .unwrap();
5846 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5847 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5848 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5849 .await
5850 .unwrap();
5851 historical_review(&mut state, &head);
5852 historical_review(&mut state, &head);
5853 state.status = RunStatus::Superseded;
5854 state.save_under(&f.home).unwrap();
5855 let task = f.task(std::slice::from_ref(&state));
5856 let before = serde_json::to_value(&state).unwrap();
5857 git::git(&review, &["reset", "--hard", &head])
5858 .await
5859 .unwrap();
5860 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5861 assert_eq!(report.evicted_copies, 1, "{report:?}");
5862 assert!(!review.exists());
5863 assert!(candidate.exists(), "selected implementation survives");
5864 assert!(report.source_unlinked_logical_bytes > 0);
5865 assert!(report.source_unlinked_allocated_bytes.is_some());
5866 assert!(report.source_evidence_added_allocated_bytes.is_some());
5867 assert_eq!(
5868 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5869 before
5870 );
5871 assert_eq!(
5872 crate::queue::Queue::at(f.home.join("queue"))
5873 .get(&task.id)
5874 .unwrap()
5875 .runs,
5876 task.runs
5877 );
5878 let archive = f
5879 .home
5880 .join("runs")
5881 .join(&state.id)
5882 .join("artifacts/source-evidence")
5883 .join(format!("{head}-{}", f.base));
5884 let patch = source_diff(&archive).await.unwrap();
5885 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5886 assert!(String::from_utf8_lossy(&patch).contains("100755"));
5887 assert!(
5888 git::worktree_remove_clean(&f.repo, &candidate)
5889 .await
5890 .unwrap()
5891 );
5892 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5893 .await
5894 .unwrap();
5895 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5896 .await
5897 .unwrap();
5898 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5899 assert!(
5900 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5901 .await
5902 .unwrap()
5903 .ok()
5904 );
5905 std::fs::remove_dir_all(&f.repo).unwrap();
5907 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5908 let restored = f._temp.path().join("restored");
5909 restore_source(&archive, &restored).await.unwrap();
5910 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5911 assert_eq!(
5912 std::fs::read(restored.join("binary")).unwrap(),
5913 b"new\0binary\xfe"
5914 );
5915 assert!(!restored.join("source").exists());
5916 assert_ne!(
5917 std::fs::metadata(restored.join("script"))
5918 .unwrap()
5919 .permissions()
5920 .mode()
5921 & 0o111,
5922 0
5923 );
5924 assert!(restore_source(&archive, &restored).await.is_err());
5925 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5928 .await
5929 .unwrap();
5930 let patch_file = f._temp.path().join("saved.patch");
5931 std::fs::write(&patch_file, &patch).unwrap();
5932 git::git(
5933 &restored,
5934 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5935 )
5936 .await
5937 .unwrap();
5938 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5939 assert_eq!(
5940 std::fs::read(restored.join("binary")).unwrap(),
5941 b"new\0binary\xfe"
5942 );
5943 }
5944
5945 #[tokio::test]
5946 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5947 let f = Fx::new().await;
5948 let state = inactive_source(&f, "ec10", true).await;
5949 let review = reviewer(&f, &state, &f.base).await;
5950 let mut successor = inactive_source(&f, "ec11", true).await;
5951 successor.retired_to = Some(state.id.clone());
5952 successor.save_under(&f.home).unwrap();
5953 f.task(&[state.clone(), successor]);
5954
5955 let mut tips = Vec::new();
5956 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5957 std::fs::write(review.join("source"), contents).unwrap();
5958 git::git(&review, &["add", "."]).await.unwrap();
5959 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5960 let tip = git::git(
5961 &review,
5962 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5963 )
5964 .await
5965 .unwrap();
5966 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5967 tips.push(tip);
5968 git::git(&review, &["reset", "--hard", &f.base])
5969 .await
5970 .unwrap();
5971 }
5972 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5974 .await
5975 .unwrap();
5976 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5977 .await
5978 .unwrap();
5979 let mut report = Report::default();
5980 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5981 assert_eq!(report.evicted_copies, 1, "{report:?}");
5982 assert!(!review.exists());
5983 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5984 .await
5985 .unwrap();
5986 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5987 for tip in &tips {
5988 assert!(
5989 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5990 .await
5991 .unwrap()
5992 .ok()
5993 );
5994 }
5995 std::fs::remove_dir_all(&f.repo).unwrap();
5997 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
5998 for (i, tip) in tips.iter().enumerate() {
5999 assert!(
6000 shown
6001 .iter()
6002 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
6003 );
6004 let destination = f._temp.path().join(format!("restore-history-{i}"));
6005 restore_seat(&f.home, &state.id, tip, &destination)
6006 .await
6007 .unwrap();
6008 assert_eq!(
6009 std::fs::read_to_string(destination.join("source")).unwrap(),
6010 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
6011 );
6012 }
6013 assert!(
6014 restore_seat(
6015 &f.home,
6016 &state.id,
6017 "unknown",
6018 &f._temp.path().join("missing")
6019 )
6020 .await
6021 .is_err()
6022 );
6023 }
6024
6025 #[tokio::test]
6026 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
6027 fn corrupt(path: &Path) -> Result<()> {
6028 let root = path.parent().unwrap();
6029 let marker = root.join("archive-to-corrupt");
6032 let archive = std::fs::read_to_string(&marker)?;
6033 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
6034 Ok(())
6035 }
6036 let f = Fx::new().await;
6037 let state = inactive_source(&f, "ec12", true).await;
6038 let review = reviewer(&f, &state, &f.base).await;
6039 let archive = f
6040 .home
6041 .join("runs")
6042 .join(&state.id)
6043 .join("artifacts/source-evidence")
6044 .join(format!("{}-{}", f.base, f.base));
6045 std::fs::write(
6046 state.worktree_root().join("archive-to-corrupt"),
6047 archive.to_str().unwrap(),
6048 )
6049 .unwrap();
6050 let mut report = Report::default();
6051 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
6052 assert_eq!(report.evicted_copies, 0, "{report:?}");
6053 assert!(review.exists());
6054 assert!(
6055 report
6056 .retained
6057 .iter()
6058 .any(|r| r.contains("corrupt snapshot")),
6059 "{report:?}"
6060 );
6061 }
6062
6063 #[tokio::test]
6064 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6065 for kind in [
6066 "dirty",
6067 "hidden",
6068 "ignored",
6069 "empty-directory",
6070 "untracked",
6071 "borrowed",
6072 "borrowed-symlink",
6073 "lease",
6074 "owner",
6075 "driver",
6076 "task-pin",
6077 "task-replacement",
6078 "malformed-orig-head",
6079 "grace",
6080 "resumable",
6081 "archive-symlink",
6082 "root-symlink",
6083 "checkpoint",
6084 "symlink",
6085 "retained",
6086 "unreadable",
6087 "unknown-field",
6088 "unknown-task-field",
6089 ] {
6090 let f = Fx::new().await;
6091 let mut state = inactive_source(&f, "ec02", true).await;
6092 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6093 .await
6094 .unwrap();
6095 historical_review(&mut state, &head);
6096 let review = reviewer(&f, &state, &head).await;
6097 let mut output = None;
6098 let mut owner = None;
6099 let mut driver = None;
6100 let mut disk = Fx::disk();
6101 match kind {
6102 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6103 "hidden" => {
6104 git::git(&review, &["update-index", "--assume-unchanged", "source"])
6105 .await
6106 .unwrap();
6107 std::fs::write(review.join("source"), "notes").unwrap();
6108 }
6109 "ignored" => {
6110 std::fs::create_dir(review.join("scratch")).unwrap();
6111 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6112 }
6113 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6114 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6115 "borrowed" => {
6116 let mut other = f.run("ec03", true).await;
6117 other.candidates[0].worktree = review.clone();
6118 other.save_under(&f.home).unwrap();
6119 }
6120 "borrowed-symlink" => {
6121 let alias = f._temp.path().join("borrowed-alias");
6122 std::os::unix::fs::symlink(&review, &alias).unwrap();
6123 let mut other = f.run("ec03", true).await;
6124 other.candidates[0].worktree = alias;
6125 other.save_under(&f.home).unwrap();
6126 }
6127 "lease" => output = OutputLease::borrow(&review).unwrap(),
6128 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6129 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6130 "task-pin" => {
6131 let mut task = f.task(std::slice::from_ref(&state));
6132 task.resume_override = Some(crate::queue::OperatorResume {
6133 question_id: "fixture-pin".into(),
6134 at: Fx::now(),
6135 conductor_rehold: None,
6136 forced: true,
6137 pinned_run: Some(state.id.clone()),
6138 });
6139 crate::queue::Queue::at(f.home.join("queue"))
6140 .put(&mut task)
6141 .unwrap();
6142 }
6143 "task-replacement" => {
6144 let mut task = f.task(std::slice::from_ref(&state));
6145 task.replacement = Some(crate::merge_decision::Replacement {
6146 question: "fixture-replacement".into(),
6147 run: state.id.clone(),
6148 pr: "https://example.invalid/pull/1".into(),
6149 agent: "worker".into(),
6150 context: String::new(),
6151 });
6152 crate::queue::Queue::at(f.home.join("queue"))
6153 .put(&mut task)
6154 .unwrap();
6155 }
6156 "malformed-orig-head" => {
6157 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6158 .await
6159 .unwrap();
6160 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6161 .unwrap();
6162 }
6163 "grace" => disk.fold_grace_secs = 3600,
6164 "resumable" => state.status = RunStatus::Blocked,
6165 "archive-symlink" => {
6166 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6167 std::fs::create_dir_all(&artifacts).unwrap();
6168 let outside = f._temp.path().join("outside");
6169 std::fs::create_dir(&outside).unwrap();
6170 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6171 .unwrap();
6172 }
6173 "root-symlink" => {
6174 let root = state.worktree_root();
6175 let outside = f._temp.path().join("outside-root");
6176 std::fs::rename(&root, &outside).unwrap();
6177 std::os::unix::fs::symlink(&outside, &root).unwrap();
6178 }
6179 "checkpoint" => {
6180 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6181 .await
6182 .unwrap();
6183 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6184 }
6185 "symlink" => {
6186 std::fs::remove_file(review.join("source")).unwrap();
6187 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6188 }
6189 "retained" => std::fs::write(
6190 f.home
6191 .join("runs")
6192 .join(&state.id)
6193 .join("retain-recovery-sources"),
6194 "owner",
6195 )
6196 .unwrap(),
6197 "unreadable" => {
6198 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6199 }
6200 "unknown-field" | "unknown-task-field" => {}
6201 _ => unreachable!(),
6202 }
6203 state.save_under(&f.home).unwrap();
6204 if kind == "unknown-field" {
6205 let path = f.home.join("runs").join(&state.id).join("run.json");
6206 let mut raw: serde_json::Value =
6207 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6208 raw["future_source_owner"] = serde_json::json!({"path": review});
6209 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6210 }
6211 if kind == "unknown-task-field" {
6212 let task = f.task(std::slice::from_ref(&state));
6213 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6214 let mut raw: serde_json::Value =
6215 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6216 raw["future_source_owner"] = serde_json::json!({"path": review});
6217 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6218 }
6219 let mut report = Report::default();
6220 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6221 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6222 assert!(review.exists(), "{kind}: {report:?}");
6223 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6224 drop(output);
6225 drop(owner);
6226 drop(driver);
6227 }
6228 }
6229
6230 #[tokio::test]
6231 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6232 let f = Fx::new().await;
6233 let state = inactive_source(&f, "ec04", true).await;
6234 let review = reviewer(&f, &state, &f.base).await;
6235 let mut report = Report::default();
6236 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6237 assert_eq!(report.evicted_copies, 1, "{report:?}");
6238 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6239 let ledger = artifacts.join("eviction-review-1.json");
6240 let mut record: serde_json::Value =
6241 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6242 record["phase"] = "pending".into();
6243 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6244 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6245 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6246 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6247 let mut blocked = Report::default();
6248 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6249 assert!(
6250 blocked
6251 .retained
6252 .iter()
6253 .any(|r| r.contains("corrupt snapshot")),
6254 "{blocked:?}"
6255 );
6256 assert_eq!(
6257 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6258 ["phase"],
6259 "pending"
6260 );
6261 assert!(
6262 restore_source(&archive, &f._temp.path().join("bad-restore"))
6263 .await
6264 .is_err()
6265 );
6266 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6267 let mut retried = Report::default();
6268 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6269 assert_eq!(
6270 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6271 ["phase"],
6272 "evicted"
6273 );
6274 assert_eq!(retried.evicted_copies, 0);
6275 assert!(!review.exists());
6276 let pending = archive.with_extension("pending");
6278 std::fs::rename(&archive, &pending).unwrap();
6279 assert_eq!(
6280 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6281 .await
6282 .unwrap(),
6283 archive
6284 );
6285 assert!(!pending.exists());
6286 }
6287
6288 #[tokio::test]
6289 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6290 let f = Fx::new().await;
6291 let mut empty = inactive_source(&f, "ec05", false).await;
6292 historical_review(&mut empty, &f.base);
6293 empty.status = RunStatus::Superseded;
6294 empty.save_under(&f.home).unwrap();
6295 let source = inactive_source(&f, "ec06", true).await;
6296 let task = f.task(&[empty.clone(), source.clone()]);
6297 let before = serde_json::to_value(&empty).unwrap();
6298 let mut report = Report::default();
6299 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6300 assert_eq!(report.evicted_copies, 1, "{report:?}");
6301 assert!(!empty.candidates[0].worktree.exists());
6302 assert!(source.candidates[0].worktree.exists());
6303 assert!(
6304 git::branch_exists(&f.repo, &empty.candidates[0].branch)
6305 .await
6306 .unwrap()
6307 );
6308 assert_eq!(
6309 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6310 before
6311 );
6312 let mut retry = Report::default();
6313 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6314 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6315 assert!(!empty.worktree_root().exists());
6316 assert_eq!(
6317 crate::queue::Queue::at(f.home.join("queue"))
6318 .get(&task.id)
6319 .unwrap()
6320 .runs,
6321 task.runs
6322 );
6323 }
6324 #[tokio::test]
6325 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6326 fn interrupt(_: &Path) -> Result<()> {
6327 anyhow::bail!("fixture interruption before unlink")
6328 }
6329 fn vanish(path: &Path) -> Result<()> {
6330 std::fs::remove_dir_all(path)?;
6331 anyhow::bail!("fixture interruption after checkout unlink")
6332 }
6333 for stage in [
6334 "intent",
6335 "missing-registration",
6336 "changed-admin",
6337 "missing-archive",
6338 "wrong-identity",
6339 ] {
6340 let f = Fx::new().await;
6341 let state = inactive_source(&f, "ec07", true).await;
6342 let review = reviewer(&f, &state, &f.base).await;
6343 let admin = PathBuf::from(
6344 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6345 .await
6346 .unwrap(),
6347 );
6348 let mut interrupted = Report::default();
6349 copies::evict_with(
6350 &f.home,
6351 &Fx::disk(),
6352 Fx::now(),
6353 &mut interrupted,
6354 if stage == "missing-registration" || stage == "changed-admin" {
6355 vanish
6356 } else {
6357 interrupt
6358 },
6359 )
6360 .await;
6361 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6362 let ledger = f
6363 .home
6364 .join("runs")
6365 .join(&state.id)
6366 .join("artifacts/eviction-review-1.json");
6367 let record: serde_json::Value =
6368 serde_json::from_slice(&std::fs::read(&ledger).unwrap_or_else(|error| {
6369 panic!("{stage}: missing pending ledger: {error}; {interrupted:?}")
6370 }))
6371 .unwrap();
6372 assert_eq!(record["phase"], "pending");
6373 if stage == "changed-admin" {
6374 std::fs::write(admin.join("owner-note"), "keep").unwrap();
6375 }
6376 if stage == "missing-archive" {
6377 std::fs::remove_file(
6378 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6379 )
6380 .unwrap();
6381 }
6382 if stage == "wrong-identity" {
6383 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6384 .await
6385 .unwrap();
6386 let other =
6387 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6388 .await
6389 .unwrap();
6390 let original = Path::new(record["evidence"].as_str().unwrap());
6391 for name in [
6392 "manifest.json",
6393 "objects.pack",
6394 "change.patch",
6395 "change.index",
6396 ] {
6397 std::fs::copy(other.join(name), original.join(name)).unwrap();
6398 }
6399 }
6400 let mut retried = Report::default();
6401 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6402 let phase =
6403 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6404 .unwrap()["phase"]
6405 .clone();
6406 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6407 assert_eq!(phase, "pending", "{stage}: {retried:?}");
6408 assert!(!retried.retained.is_empty());
6409 if stage == "changed-admin" {
6410 assert!(admin.join("owner-note").exists());
6411 } else {
6412 assert!(review.exists());
6413 }
6414 } else {
6415 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6416 assert!(!review.exists());
6417 assert!(!admin.exists());
6418 }
6419 }
6420 }
6421 #[tokio::test]
6422 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
6423 let f = Fx::new().await;
6424 let mut runs = Vec::new();
6425 let mut paths = Vec::new();
6426 for tag in ["ec10", "ec11", "ec12", "ec13"] {
6427 let state = inactive_source(&f, tag, true).await;
6428 for name in ["judge-1", "advisor-1"] {
6429 let path = state.worktree_root().join(name);
6430 git::worktree_add_detached(&f.repo, &path, &f.base)
6431 .await
6432 .unwrap();
6433 paths.push(path);
6434 }
6435 runs.push(state);
6436 }
6437 let task = f.task(&runs);
6438 let mut report = Report::default();
6439 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6440 assert_eq!(report.evicted_copies, 5, "{report:?}");
6441 let mut next = Report::default();
6442 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6443 assert_eq!(next.evicted_copies, 3, "{next:?}");
6444 assert!(paths.iter().all(|p| !p.exists()));
6445 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
6446 assert_eq!(
6447 crate::queue::Queue::at(f.home.join("queue"))
6448 .get(&task.id)
6449 .unwrap()
6450 .runs,
6451 task.runs
6452 );
6453 let limited = Fx::new().await;
6454 let state = inactive_source(&limited, "ec15", true).await;
6455 let copy = reviewer(&limited, &state, &limited.base).await;
6456 let evidence = limited
6457 .home
6458 .join("runs")
6459 .join(&state.id)
6460 .join("artifacts/source-evidence");
6461 std::fs::create_dir_all(&evidence).unwrap();
6462 let retained = evidence.join("reserved-evidence");
6463 std::fs::File::create(&retained)
6464 .unwrap()
6465 .set_len(64 * 1024 * 1024)
6466 .unwrap();
6467 let mut blocked = Report::default();
6468 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6469 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
6470 assert!(copy.exists());
6471 assert_eq!(
6472 std::fs::metadata(&retained).unwrap().len(),
6473 64 * 1024 * 1024
6474 );
6475 assert!(
6476 blocked
6477 .retained
6478 .iter()
6479 .any(|r| r.contains("per-run storage limit")),
6480 "{blocked:?}"
6481 );
6482 }
6483 #[tokio::test]
6484 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
6485 let f = Fx::new().await;
6486 let mut state = inactive_source(&f, "ec14", true).await;
6487 historical_review(&mut state, &f.base);
6488 historical_review(&mut state, &f.base);
6489 state.status = RunStatus::Superseded;
6490 state.save_under(&f.home).unwrap();
6491 let review = reviewer(&f, &state, &f.base).await;
6492 let task = f.task(std::slice::from_ref(&state));
6493 let path = f.home.join("runs").join(&state.id).join("run.json");
6494 let mut raw: serde_json::Value =
6495 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6496 raw["schema"] = 10.into();
6497 raw.as_object_mut().unwrap().remove("output_activity_at");
6498 raw["resource_wait"] = serde_json::Value::Null;
6501 raw["resource_block"] = serde_json::Value::Null;
6502 raw["driver_claimed"] = serde_json::Value::Null;
6503 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6504 let graph = raw["config"]["graph"].as_object_mut().unwrap();
6506 let count = graph.remove("implementers").unwrap();
6507 graph.insert("candidates".into(), count);
6508 let queue = crate::queue::Queue::at(f.home.join("queue"));
6509 let task_path = queue.path_of(&task.id);
6510 let mut task_raw: serde_json::Value =
6511 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
6512 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6513 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
6514 std::fs::write(&task_path, &task_historical).unwrap();
6515 let historical = serde_json::to_vec_pretty(&raw).unwrap();
6516 std::fs::write(&path, &historical).unwrap();
6517 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6518 assert_eq!(report.evicted_copies, 1, "{report:?}");
6519 assert!(!review.exists());
6520 assert_eq!(std::fs::read(&path).unwrap(), historical);
6521 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
6522 assert!(state.candidates[0].worktree.exists());
6523 assert_eq!(
6524 crate::queue::Queue::at(f.home.join("queue"))
6525 .get(&task.id)
6526 .unwrap()
6527 .runs,
6528 task.runs
6529 );
6530 }
6531 #[tokio::test]
6532 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
6533 let f = Fx::new().await;
6534 let state = inactive_source(&f, "ed01", true).await;
6535 let review = reviewer(&f, &state, &f.base).await;
6536 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
6537 git::git(&review, &["add", "."]).await.unwrap();
6538 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6539 let tip = git::git(
6540 &review,
6541 &[
6542 "commit-tree",
6543 &tree,
6544 "-p",
6545 &f.base,
6546 "-m",
6547 "temporary detached source",
6548 ],
6549 )
6550 .await
6551 .unwrap();
6552 git::git(&review, &["checkout", "--detach", &tip])
6553 .await
6554 .unwrap();
6555 git::git(&review, &["checkout", "--detach", &f.base])
6556 .await
6557 .unwrap();
6558 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
6559 .await
6560 .unwrap();
6561 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
6562 .await
6563 .unwrap();
6564 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
6565 .await
6566 .unwrap();
6567 assert!(!shown.lines().any(|oid| oid == tip));
6568 let admin = PathBuf::from(
6569 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6570 .await
6571 .unwrap(),
6572 );
6573 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
6574 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
6575 fn interrupt(_: &Path) -> Result<()> {
6576 anyhow::bail!("fixture interruption before unlink")
6577 }
6578 let mut pending = Report::default();
6579 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
6580 assert!(review.exists());
6581 let ledger = f
6582 .home
6583 .join("runs")
6584 .join(&state.id)
6585 .join("artifacts/eviction-review-1.json");
6586 let complete = std::fs::read(&ledger).unwrap();
6587 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
6588 incomplete["history"]
6589 .as_object_mut()
6590 .unwrap()
6591 .remove(&tip)
6592 .unwrap();
6593 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
6594 let mut blocked = Report::default();
6595 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6596 assert!(review.exists());
6597 assert!(
6598 blocked
6599 .retained
6600 .iter()
6601 .any(|reason| reason.contains("incomplete reflog checkpoint")),
6602 "{blocked:?}"
6603 );
6604 std::fs::write(&ledger, complete).unwrap();
6605 let mut report = Report::default();
6606 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6607 assert_eq!(report.evicted_copies, 1, "{report:?}");
6608 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6609 .await
6610 .unwrap();
6611 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6612 assert!(
6613 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
6614 .await
6615 .unwrap()
6616 .ok()
6617 );
6618 std::fs::remove_dir_all(&f.repo).unwrap();
6619 let destination = f._temp.path().join("restore-old-only");
6620 restore_seat(&f.home, &state.id, &tip, &destination)
6621 .await
6622 .unwrap();
6623 assert_eq!(
6624 std::fs::read(destination.join("old-only")).unwrap(),
6625 b"old-only\0binary\xff"
6626 );
6627 }
6628
6629 #[tokio::test]
6630 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
6631 fn interrupt(_: &Path) -> Result<()> {
6632 anyhow::bail!("fixture interruption before unlink")
6633 }
6634 fn vanish(path: &Path) -> Result<()> {
6635 std::fs::remove_dir_all(path)?;
6636 anyhow::bail!("fixture interruption before unregistration")
6637 }
6638 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
6639 let f = Fx::new().await;
6640 let mut state = inactive_source(&f, "ed02", true).await;
6641 state.driver_exited = false;
6642 state.driver_claimed = Some(true);
6643 state.driver_pid = Some(std::process::id());
6644 state.driver_started_at = crate::proc::process_started_at(std::process::id());
6645 state.begin_pass();
6646 state.status = RunStatus::Superseded;
6647 state.save_under(&f.home).unwrap();
6648 let review = reviewer(&f, &state, &f.base).await;
6649 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
6652 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
6653 std::fs::write(
6654 f.home.join("runs").join(&state.id).join("driver.json"),
6655 serde_json::to_vec(&identity).unwrap(),
6656 )
6657 .unwrap();
6658 let live = if stage == "live-driver" {
6659 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
6660 } else {
6661 None
6662 };
6663 assert_eq!(
6664 state.liveness_under(&f.home, false),
6665 if live.is_some() {
6666 Liveness::Live
6667 } else {
6668 Liveness::Dead
6669 }
6670 );
6671 let mut report = Report::default();
6672 if stage == "intent" || stage == "missing-registration" {
6673 copies::evict_with(
6674 &f.home,
6675 &Fx::disk(),
6676 Fx::now(),
6677 &mut report,
6678 if stage == "intent" { interrupt } else { vanish },
6679 )
6680 .await;
6681 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
6682 let mut retry = Report::default();
6683 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6684 assert!(!review.exists(), "{stage}: {retry:?}");
6685 let ledger = f
6686 .home
6687 .join("runs")
6688 .join(&state.id)
6689 .join("artifacts/eviction-review-1.json");
6690 let receipt: serde_json::Value =
6691 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
6692 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
6693 } else {
6694 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6695 assert_eq!(
6696 report.evicted_copies,
6697 usize::from(live.is_none()),
6698 "{stage}: {report:?}"
6699 );
6700 assert_eq!(review.exists(), live.is_some());
6701 }
6702 drop(live);
6703 assert!(
6704 !RunState::load_under(&state.id, &f.home)
6705 .unwrap()
6706 .driver_exited
6707 );
6708 }
6709 }
6710}