1mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7 let artifacts = home.join("runs").join(&state.id).join("artifacts");
8 if !artifacts.try_exists()? {
9 return Ok(false);
10 }
11 ensure!(
12 plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13 "noncanonical artifacts"
14 );
15 for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16 ensure!(i < 4096, "eviction record inspection limit exceeded");
17 let name = entry?.file_name();
18 let name = name.to_string_lossy();
19 if name.starts_with("eviction-") && name.ends_with(".json") {
20 return Ok(true);
21 }
22 }
23 Ok(false)
24}
25
26pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29 copies::restore_execution_seats(home, state).await
30}
31
32pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34 crate::fold_evidence::validate_snapshot(evidence).await?;
35 Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40 crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
45 copies::seat_evidence(home, id).await
46}
47
48pub async fn seat_heads(
50 home: &Path,
51 id: &str,
52) -> Result<std::collections::BTreeMap<String, String>> {
53 copies::seat_heads(home, id).await
54}
55
56pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
58 copies::seat_diff(home, id, head).await
59}
60
61pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
63 copies::restore_seat(home, id, head, destination).await
64}
65
66use crate::{
67 git,
68 proc::Quiet as _,
69 run::{Liveness, RunState},
70};
71use anyhow::{Context, Result, ensure};
72use jiff::Timestamp;
73use serde::{Deserialize, Serialize};
74use std::{
75 fs::{File, OpenOptions},
76 path::{Path, PathBuf},
77};
78
79#[derive(Debug)]
83pub(crate) struct OutputLease {
84 _file: std::sync::Arc<OutputFile>,
85 _activity: bool,
86}
87
88#[derive(Debug)]
89struct OutputFile(File);
90
91impl std::ops::Deref for OutputFile {
92 type Target = File;
93 fn deref(&self) -> &File {
94 &self.0
95 }
96}
97
98#[cfg(unix)]
99impl Drop for OutputFile {
100 fn drop(&mut self) {
101 use std::os::fd::AsRawFd;
102 unsafe {
104 libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
105 }
106 }
107}
108
109impl OutputLease {
110 #[cfg(unix)]
111 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
112 use std::os::fd::AsRawFd;
113 self._file.as_raw_fd()
114 }
115
116 pub(crate) fn pre_clean(dir: &Path) -> Result<Self> {
118 let file = OpenOptions::new()
119 .create(true)
120 .truncate(false)
121 .read(true)
122 .write(true)
123 .open(dir.join("magi-pre-clean.lock"))?;
124 crate::pr_head::try_lock_file(&file)
125 .context("pre-clean already running; retry after descendants terminate")?;
126 Ok(Self {
127 _file: std::sync::Arc::new(OutputFile(file)),
128 _activity: false,
129 })
130 }
131
132 pub(crate) fn duplicate(&self) -> Self {
133 Self {
134 _file: std::sync::Arc::clone(&self._file),
135 _activity: false,
136 }
137 }
138
139 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
140 if !worktree.join(".git").is_file() {
141 return Ok(None);
142 }
143 Self::take(worktree, false).map(Some)
144 }
145 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
146 use std::hash::{Hash, Hasher};
147 let mut hash = std::collections::hash_map::DefaultHasher::new();
148 let mut normalized = cache.to_owned();
149 let mut missing = Vec::new();
150 while !normalized.exists() {
151 let Some(name) = normalized.file_name() else {
152 break;
153 };
154 missing.push(name.to_owned());
155 if !normalized.pop() {
156 break;
157 }
158 }
159 normalized = normalized.canonicalize().unwrap_or(normalized);
160 for part in missing.into_iter().rev() {
161 normalized.push(part);
162 }
163 normalized.hash(&mut hash);
164 let dir = home.join("cache-leases");
165 std::fs::create_dir_all(&dir)?;
166 Self::lock(
167 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
168 exclusive,
169 )
170 }
171 #[cfg(unix)]
175 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
176 Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
177 }
178
179 #[cfg(unix)]
180 pub(crate) fn admit_with(
181 home: &Path,
182 paths: &[PathBuf],
183 min_free: u64,
184 free_bytes: impl Fn(&Path) -> Result<u64>,
185 ) -> Result<Vec<Self>> {
186 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
187 if min_free == 0 {
188 return Ok(Vec::new());
189 }
190 const RESERVE: u64 = 1024 * 1024 * 1024;
191 std::fs::create_dir_all(home)?;
192 let dir = home.canonicalize()?.join("disk-admission");
193 std::fs::create_dir_all(&dir)?;
194 ensure!(
195 plain_dir(&dir) && dir.canonicalize()? == dir,
196 "disk admission directory is not canonical"
197 );
198 let mut devices = std::collections::BTreeSet::new();
199 let mut reservations = Vec::new();
200 for path in paths {
201 let probe = path
202 .ancestors()
203 .find(|p| p.exists())
204 .context("disk admission path missing")?;
205 let device = std::fs::metadata(probe)?.dev();
206 if !devices.insert(device) {
207 continue;
208 }
209 let open = |path: &Path| -> Result<File> {
210 Ok(OpenOptions::new()
211 .read(true)
212 .write(true)
213 .create(true)
214 .truncate(false)
215 .custom_flags(libc::O_NOFOLLOW)
216 .open(path)?)
217 };
218 let admission = Self {
222 _file: std::sync::Arc::new(OutputFile(open(&dir.join(format!("{device}.lock")))?)),
223 _activity: false,
224 };
225 let started = std::time::Instant::now();
226 loop {
227 match crate::pr_head::try_lock_file(&admission._file) {
228 Ok(()) => break,
229 Err(error)
230 if matches!(
231 error.kind(),
232 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
233 ) && started.elapsed() < std::time::Duration::from_secs(1) =>
234 {
235 std::thread::sleep(std::time::Duration::from_millis(5));
236 }
237 Err(error) => {
238 return Err(error)
239 .context("disk admission busy or unavailable; retry without dispatch");
240 }
241 }
242 }
243 let mut active = 0u64;
244 let mut available = None;
245 for slot in 0..1024 {
246 let file = dir.join(format!("{device}-{slot}.reserve"));
247 if !file.exists() {
248 if available.is_none() {
249 available = Some(file);
250 }
251 continue;
252 }
253 let file = open(&file)?;
254 match crate::pr_head::try_lock_file(&file) {
255 Ok(()) if available.is_none() => {
256 available = Some(dir.join(format!("{device}-{slot}.reserve")))
257 }
258 Ok(()) => {}
259 Err(_) => active = active.saturating_add(RESERVE),
260 }
261 }
262 let free = free_bytes(probe)?;
263 let required = min_free.saturating_add(active).saturating_add(RESERVE);
264 ensure!(
265 free >= required,
266 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
267 );
268 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
269 crate::pr_head::try_lock_file(&file)?;
270 reservations.push(Self {
271 _file: std::sync::Arc::new(OutputFile(file)),
272 _activity: false,
273 });
274 }
275 Ok(reservations)
276 }
277
278 #[cfg(not(unix))]
279 pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
280 Ok(Vec::new())
282 }
283
284 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
285 let out = std::process::Command::new("git")
286 .args(["rev-parse", "--absolute-git-dir"])
287 .current_dir(worktree)
288 .quiet()
289 .output()?;
290 ensure!(
291 out.status.success(),
292 "cannot identify output lease directory"
293 );
294 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
295 Self::lock(&dir.join("magi-output.lock"), exclusive)
296 }
297 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
298 let file = OpenOptions::new()
299 .create(true)
300 .truncate(false)
301 .read(true)
302 .write(true)
303 .open(path)
304 .with_context(|| format!("output lease: open {}", path.display()))?;
305 #[cfg(unix)]
306 {
307 use std::os::fd::AsRawFd;
308 let mode = if exclusive {
309 libc::LOCK_EX
310 } else {
311 libc::LOCK_SH
312 };
313 ensure!(
315 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
316 "output lease: outputs are in use or being reclaimed: {}",
317 path.display()
318 );
319 }
320 #[cfg(not(unix))]
321 let _ = exclusive;
322 if !exclusive {
323 file.set_len(1)?;
325 file.set_modified(std::time::SystemTime::now())
326 .with_context(|| format!("record output activity {}", path.display()))?;
327 }
328 Ok(Self {
329 _file: std::sync::Arc::new(OutputFile(file)),
330 _activity: !exclusive,
331 })
332 }
333}
334#[cfg(unix)]
335impl Drop for OutputLease {
336 fn drop(&mut self) {
337 if self._activity
338 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
339 {
340 let _ = self._file.set_len(2);
343 tracing::warn!("could not record output lease release activity: {error}");
344 }
345 }
346}
347
348#[derive(Default, Debug, Serialize, Deserialize)]
349#[serde(default)]
350pub(crate) struct Report {
351 pub checked_at: Option<Timestamp>,
352 pub limit_bytes: u64,
353 pub reclaim_budget_bytes: u64,
354 pub measured_bytes: u64,
356 pub freed_bytes: u64,
358 pub measured_allocated_file_bytes: Option<u64>,
359 pub proven_reclaimable_bytes: u64,
361 pub proven_reclaimable_allocated_bytes: Option<u64>,
364 pub protected_or_unproven_bytes: u64,
365 pub estimated_released_allocated_bytes: Option<u64>,
368 pub observed_home_free_change_bytes: Option<i64>,
371 pub retired: usize,
372 pub evicted_copies: usize,
373 pub removed_empty_roots: usize,
374 pub source_unlinked_logical_bytes: u64,
375 pub source_unlinked_allocated_bytes: Option<u64>,
376 pub source_evidence_added_logical_bytes: u64,
377 pub source_evidence_added_allocated_bytes: Option<u64>,
378 pub retained: Vec<String>,
379 pub failures: Vec<String>,
380}
381
382#[cfg(unix)]
383fn allocated_inventory(
384 root: &Path,
385 seen: &mut std::collections::BTreeSet<(u64, u64)>,
386) -> Result<u64> {
387 use std::os::unix::fs::MetadataExt;
388 ensure!(
389 plain_dir(root) && root.canonicalize()? == root,
390 "allocated inventory root is not canonical"
391 );
392 let mut dirs = vec![root.to_owned()];
393 let mut bytes = 0u64;
394 while let Some(dir) = dirs.pop() {
395 for entry in std::fs::read_dir(dir)? {
396 let path = entry?.path();
397 let metadata = std::fs::symlink_metadata(&path)?;
398 if metadata.is_dir() {
399 dirs.push(path);
400 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
401 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
402 }
403 }
404 }
405 Ok(bytes)
406}
407
408#[derive(Default)]
409struct ReleasedAllocation {
410 #[cfg(unix)]
411 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
412 #[cfg(unix)]
413 removed: std::collections::BTreeMap<(u64, u64), u64>,
414}
415
416impl ReleasedAllocation {
417 fn of(paths: &[PathBuf]) -> Result<Self> {
418 #[cfg(unix)]
419 {
420 use std::os::unix::fs::MetadataExt;
421 let mut files = std::collections::BTreeMap::new();
422 for path in paths {
423 let m = std::fs::symlink_metadata(path)?;
424 ensure!(m.is_file(), "allocation candidate is not regular");
425 files.insert(
426 path.clone(),
427 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
428 );
429 }
430 Ok(Self {
431 files,
432 removed: Default::default(),
433 })
434 }
435 #[cfg(not(unix))]
436 {
437 let _ = paths;
438 Ok(Self::default())
439 }
440 }
441 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
442 #[cfg(unix)]
443 {
444 let mut inodes = std::collections::BTreeMap::new();
445 for &(device, inode, links, blocks) in self.files.values() {
446 let entry = inodes
447 .entry((device, inode))
448 .or_insert((0u64, links, blocks));
449 entry.0 += 1;
450 }
451 Some(
452 inodes
453 .values()
454 .filter(|(count, links, _)| count == links)
455 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
456 )
457 }
458 #[cfg(not(unix))]
459 {
460 None
461 }
462 }
463
464 fn unlinked(&mut self, path: &Path) -> Option<u64> {
465 #[cfg(unix)]
466 {
467 let (device, inode, links, blocks) = self.files[path];
468 let removed = self.removed.entry((device, inode)).or_default();
469 *removed += 1;
470 Some(if *removed == links { blocks } else { 0 })
471 }
472 #[cfg(not(unix))]
473 {
474 let _ = path;
475 None
476 }
477 }
478}
479
480fn plain_dir(path: &Path) -> bool {
481 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
482}
483
484fn run_paths_missing(state: &RunState) -> Result<bool> {
487 for path in [state.worktree_root(), state.repo.clone()] {
488 match std::fs::metadata(&path) {
489 Ok(_) => {}
490 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
491 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
492 }
493 }
494 Ok(false)
495}
496
497async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
499 let root = state.worktree_root();
500 if !plain_dir(&root) {
501 return Ok(Vec::new());
502 }
503 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
504 registered_worktrees(state, &list)
505}
506
507fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
508 let root = state.worktree_root();
509 if !plain_dir(&root) {
510 return Ok(Vec::new());
511 }
512 let root = root.canonicalize()?;
513 let mut paths = Vec::new();
514 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
515 let path = PathBuf::from(path);
516 #[cfg(windows)]
517 let path = if plain_dir(&path) {
518 git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
519 } else {
520 continue;
521 };
522 if plain_dir(&path)
523 && path.parent() == Some(root.as_path())
524 && path.canonicalize()? == path
525 && path.join(".git").is_file()
526 {
527 paths.push(path);
528 }
529 }
530 Ok(paths)
531}
532
533async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
536 validate_cargo_target(worktree, false).await
537}
538
539async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
540 let target = worktree.join("target");
541 ensure!(plain_dir(&target), "target absent or not a plain directory");
542 ensure!(target.canonicalize()? == target, "target path changed");
543 ensure!(
544 git::git(worktree, &["ls-files", "--", "target"])
545 .await?
546 .is_empty(),
547 "tracked target contents"
548 );
549 ensure!(
550 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
551 .await?
552 .ok(),
553 "target is not ignored"
554 );
555 cache_tag(&target, allow_missing_tag)?;
556 let path = target.join(".rustc_info.json");
557 plain_file(&path, "rustc identity")?;
558 let info: serde_json::Value = serde_json::from_slice(
559 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
560 )
561 .with_context(|| format!("parse rustc identity {}", path.display()))?;
562 ensure!(
563 info.get("rustc_fingerprint").is_some(),
564 "unknown Cargo identity {}",
565 path.display()
566 );
567 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
568 ensure!(
569 info.get("rustc_fingerprint")
570 .is_some_and(serde_json::Value::is_u64)
571 && info
572 .get("outputs")
573 .and_then(serde_json::Value::as_object)
574 .is_some_and(|outputs| outputs.values().any(|output| {
575 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
576 && output
577 .get("stdout")
578 .and_then(serde_json::Value::as_str)
579 .is_some_and(|stdout| stdout.starts_with("rustc "))
580 && output
581 .get("stderr")
582 .is_some_and(serde_json::Value::is_string)
583 }))
584 && info
585 .get("successes")
586 .is_some_and(serde_json::Value::is_object),
587 "missing-tag rustc identity structure invalid: {}",
588 path.display()
589 );
590 let mut profiles = 0;
591 for profile in ["debug", "release"] {
592 let dir = target.join(profile);
593 if !dir.exists() {
594 continue;
595 }
596 for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
597 ensure!(
598 plain_dir(&subdir) && subdir.canonicalize()? == subdir,
599 "missing-tag profile proof: {} is not a plain directory",
600 subdir.display()
601 );
602 }
603 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
604 profiles += 1;
605 }
606 ensure!(
607 profiles > 0,
608 "missing-tag proof: no Cargo profile in {}",
609 target.display()
610 );
611 }
612 for name in ["debug", "release"] {
613 let path = target.join(name);
614 match std::fs::symlink_metadata(&path) {
615 Ok(metadata) => ensure!(
616 metadata.is_dir() && path.canonicalize()? == path,
617 "profile validation: not a plain directory: {}",
618 path.display()
619 ),
620 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
621 Err(error) => {
622 return Err(error)
623 .with_context(|| format!("profile validation: inspect {}", path.display()));
624 }
625 }
626 }
627 Ok(target)
628}
629
630fn plain_file(path: &Path, stage: &str) -> Result<()> {
631 let meta = std::fs::symlink_metadata(path)
632 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
633 ensure!(
634 meta.is_file() && path.canonicalize()? == path,
635 "{stage}: not a plain file: {}",
636 path.display()
637 );
638 Ok(())
639}
640
641fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
642 let path = target.join("CACHEDIR.TAG");
643 match std::fs::symlink_metadata(&path) {
644 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
645 return Ok(());
646 }
647 result => {
648 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
649 }
650 }
651 plain_file(&path, "cache tag")?;
652 let tag = std::fs::read_to_string(&path)
653 .with_context(|| format!("cache tag: read {}", path.display()))?;
654 ensure!(
655 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
656 "unknown Cargo cache tag: {}",
657 path.display()
658 );
659 Ok(())
660}
661
662fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
666 let mut dirs = vec![target.to_owned()];
667 let mut entries = 0;
668 while let Some(dir) = dirs.pop() {
669 for entry in std::fs::read_dir(&dir)
670 .with_context(|| format!("output activity: list {}", dir.display()))?
671 {
672 let path = entry?.path();
673 entries += 1;
674 ensure!(
675 entries <= 100_000,
676 "output activity inventory too large: {}",
677 target.display()
678 );
679 let metadata = std::fs::symlink_metadata(&path)
680 .with_context(|| format!("output activity: inspect {}", path.display()))?;
681 if metadata.is_dir() {
682 if dir != target
684 || matches!(
685 path.file_name().and_then(|s| s.to_str()),
686 Some("debug" | "release")
687 )
688 {
689 dirs.push(path);
690 }
691 } else if metadata.is_file()
692 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
693 {
694 let at =
695 Timestamp::try_from(metadata.modified().with_context(|| {
696 format!("output activity: modified {}", path.display())
697 })?)?;
698 ensure!(
699 crate::clean::due(now, at, grace),
700 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
701 path.display()
702 );
703 }
704 }
705 }
706 Ok(())
707}
708
709pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
710 use std::io::Read as _;
711 let mut header = [0; 8];
712 let n = File::open(path)?.read(&mut header)?;
713 let header = &header[..n];
714 Ok(header.starts_with(b"\x7fELF")
715 || header.starts_with(b"!<arch>\n")
716 || header.starts_with(b"rust\0\0\0")
717 || header.starts_with(b"MZ\x90\0")
718 || [
719 [0xfe, 0xed, 0xfa, 0xce],
720 [0xce, 0xfa, 0xed, 0xfe],
721 [0xfe, 0xed, 0xfa, 0xcf],
722 [0xcf, 0xfa, 0xed, 0xfe],
723 [0xca, 0xfe, 0xba, 0xbe],
724 [0xca, 0xfe, 0xba, 0xbf],
725 ]
726 .iter()
727 .any(|magic| header.starts_with(magic)))
728}
729
730fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
734 let mut outputs = std::collections::BTreeSet::new();
735 for profile in ["debug", "release"] {
736 for dir in [target.join(profile), target.join(profile).join("deps")] {
737 if !plain_dir(&dir) || dir.canonicalize()? != dir {
738 continue;
739 }
740 for entry in std::fs::read_dir(&dir)? {
741 let path = entry?.path();
742 if path.extension().is_none_or(|ext| ext != "d")
743 || !std::fs::symlink_metadata(&path)?.is_file()
744 || std::fs::metadata(&path)?.len() > 1024 * 1024
745 || path.canonicalize()? != path
746 {
747 continue;
748 }
749 let body = std::fs::read_to_string(&path)?;
750 for line in body.lines() {
751 let Some((names, dependencies)) = line.split_once(": ") else {
752 continue;
753 };
754 if dependencies.trim().is_empty() {
755 continue;
756 }
757 let mut words = Vec::new();
760 let mut word = String::new();
761 let mut escaped = false;
762 for c in names.chars() {
763 if escaped {
764 word.push(c);
765 escaped = false;
766 } else if c == '\\' {
767 escaped = true;
768 } else if c.is_whitespace() {
769 if !word.is_empty() {
770 words.push(std::mem::take(&mut word));
771 }
772 } else {
773 word.push(c);
774 }
775 }
776 if escaped || word.contains(['$', '%']) {
777 continue;
778 }
779 if !word.is_empty() {
780 words.push(word);
781 }
782 for name in words {
783 let output = PathBuf::from(name);
784 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
785 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
786 let artifact_name =
787 output_stem == stem || output_stem == format!("lib{stem}");
788 let artifact_kind = output.extension().is_none_or(|ext| {
789 matches!(
790 ext.to_str(),
791 Some(
792 "rlib"
793 | "rmeta"
794 | "so"
795 | "dylib"
796 | "a"
797 | "lib"
798 | "dll"
799 | "exe"
800 | "pdb"
801 )
802 )
803 });
804 if artifact_name
805 && artifact_kind
806 && output.parent() == Some(dir.as_path())
807 && output.extension().is_none_or(|ext| ext != "d")
808 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
809 && output.canonicalize().is_ok_and(|p| p == output)
810 && artifact_signature(&output)?
811 {
812 outputs.insert(output);
813 }
814 }
815 }
816 }
817 }
818 }
819 Ok(outputs.into_iter().collect())
820}
821
822#[derive(Serialize, Deserialize)]
823struct CargoResidue {
824 target: PathBuf,
825 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
826}
827
828fn residue_files(
831 target: &Path,
832 outputs: &[PathBuf],
833) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
834 let mut files = std::collections::BTreeMap::new();
835 let mut dirs = vec![target.to_owned()];
836 let mut bytes = 0usize;
837 let mut entries = 0usize;
838 while let Some(dir) = dirs.pop() {
839 for entry in std::fs::read_dir(dir)? {
840 let path = entry?.path();
841 entries += 1;
842 ensure!(entries <= 4096, "residue inventory too large");
843 let metadata = std::fs::symlink_metadata(&path)?;
844 ensure!(
845 !metadata.file_type().is_symlink(),
846 "symlink in Cargo residue"
847 );
848 if metadata.is_dir() {
849 dirs.push(path);
850 } else {
851 ensure!(metadata.is_file(), "non-file in Cargo residue");
852 if outputs.contains(&path) {
853 continue;
854 }
855 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
856 let body = std::fs::read(&path)?;
857 bytes = bytes.saturating_add(body.len());
858 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
859 files.insert(path.strip_prefix(target)?.to_owned(), body);
860 }
861 }
862 }
863 Ok(files)
864}
865
866fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
867 Ok(home
868 .join("runs")
869 .join(id)
870 .join("build-residue")
871 .join(worktree.file_name().context("missing worktree name")?))
872}
873
874fn residue_root(target: &Path) -> Result<()> {
877 for entry in std::fs::read_dir(target)? {
878 let name = entry?.file_name();
879 ensure!(
880 matches!(
881 name.to_str(),
882 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
883 ),
884 "unrecognized Cargo residue root entry"
885 );
886 }
887 Ok(())
888}
889
890fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
891 cache_tag(target, false)?;
892 residue_root(target)?;
893 let files = residue_files(target, outputs)?;
894 for (relative, body) in &files {
895 let path = target.join(relative);
896 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
897 let lock = matches!(
898 relative.to_str(),
899 Some("debug/.cargo-lock" | "release/.cargo-lock")
900 ) && body.is_empty();
901 let dep_info = path.extension().is_some_and(|ext| ext == "d")
902 && outputs.iter().any(|output| {
903 output.parent() == path.parent()
904 && (output.file_stem() == path.file_stem()
905 || output.file_stem().is_some_and(|stem| {
906 stem.to_string_lossy()
907 == format!(
908 "lib{}",
909 path.file_stem().unwrap_or_default().to_string_lossy()
910 )
911 }))
912 });
913 ensure!(identity || lock || dep_info, "unproven Cargo residue");
914 }
915 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
916 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
917 let tmp = proof.with_extension("tmp");
918 std::fs::write(
919 &tmp,
920 serde_json::to_vec(&CargoResidue {
921 target: target.to_owned(),
922 files,
923 })?,
924 )?;
925 std::fs::rename(tmp, proof)?;
926 Ok(())
927}
928
929pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
930 let target = cargo_target(worktree).await?;
931 residue_root(&target)?;
932 let locks = cargo_locks(&target)?;
933 let proof: CargoResidue =
934 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
935 ensure!(
936 proof.target == target && proof.files == residue_files(&target, &[])?,
937 "Cargo residue changed or contains unproven files"
938 );
939 Ok(locks)
940}
941
942pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
944 let mut held = Vec::new();
945 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
946 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
947 }) {
948 let lock = target.join(profile).join(name);
949 match std::fs::symlink_metadata(&lock) {
950 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
951 result => {
952 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
953 }
954 }
955 {
956 plain_file(&lock, "Cargo lock")?;
957 let file = OpenOptions::new()
958 .read(true)
959 .write(true)
960 .open(&lock)
961 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
962 crate::pr_head::try_lock_file(&file)
963 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
964 held.push(file);
965 }
966 }
967 Ok(held)
968}
969
970pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
973 let home = home.to_owned();
974 let disk = disk.clone();
975 let runtime = tokio::runtime::Handle::current();
976 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
977 .await
978 .unwrap_or_else(|e| Report {
979 failures: vec![format!("retention worker failed: {e}")],
980 ..Report::default()
981 })
982}
983
984#[derive(Default, Serialize, Deserialize)]
985struct Measurement {
986 at: Option<Timestamp>,
987 bytes: u64,
988}
989
990async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
991 let mut report = Report {
992 checked_at: Some(now),
993 limit_bytes: disk.worktree_limit_bytes,
994 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
995 ..Report::default()
996 };
997 if !cfg!(unix) {
998 report
999 .failures
1000 .push("Automatic output reclamation requires Unix process containment".into());
1001 return report;
1002 }
1003 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
1004 return report;
1005 };
1006 let mut targets = Vec::new();
1007 let mut under_pressure = false;
1008 let mut registrations = std::collections::HashMap::new();
1009 let measurements_path = home.join("output-measurements.json");
1010 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
1011 std::fs::read(&measurements_path)
1012 .ok()
1013 .and_then(|b| serde_json::from_slice(&b).ok())
1014 .unwrap_or_default();
1015 let mut observed = std::collections::BTreeSet::new();
1016 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
1017 .into_iter()
1018 .flatten()
1019 .flatten()
1020 .map(|e| e.file_name().to_string_lossy().into_owned())
1021 .collect();
1022 ids.sort();
1023 for id in ids {
1024 if !crate::run::is_run_id(&id) {
1025 continue;
1026 }
1027 let state = match RunState::load_under(&id, home) {
1028 Ok(state) => state,
1029 Err(e) => {
1030 report
1031 .retained
1032 .push(format!("{id}: unreadable run ownership: {e:#}"));
1033 continue;
1034 }
1035 };
1036 if disk.worktree_limit_bytes == 0 {
1037 continue;
1038 }
1039 match run_paths_missing(&state) {
1040 Ok(true) => continue,
1041 Ok(false) => {}
1042 Err(e) => {
1043 report.failures.push(format!("{id}: {e:#}"));
1044 continue;
1045 }
1046 }
1047 let list = match registrations.entry(state.repo.clone()) {
1048 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1049 std::collections::hash_map::Entry::Vacant(entry) => {
1050 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1051 Ok(list) => entry.insert(list),
1052 Err(e) => {
1053 if matches!(run_paths_missing(&state), Ok(true)) {
1056 continue;
1057 }
1058 report.failures.push(format!("{id}: {e:#}"));
1059 continue;
1060 }
1061 }
1062 }
1063 };
1064 let paths = match registered_worktrees(&state, list) {
1065 Ok(paths) => paths,
1066 Err(e) => {
1067 report.failures.push(format!("{id}: {e:#}"));
1068 continue;
1069 }
1070 };
1071 for path in paths {
1072 if !path.join("target").exists() {
1073 continue;
1074 }
1075 let target = path.join("target");
1076 observed.insert(target.clone());
1077 let pressure =
1078 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1079 under_pressure |= pressure;
1080 let measurement = measurements.entry(target).or_default();
1081 if pressure
1082 || measurement
1083 .at
1084 .is_none_or(|at| crate::clean::due(now, at, 300))
1085 {
1086 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1087 measurement.at = Some(now);
1088 }
1089 let size = measurement.bytes;
1090 report.measured_bytes = report.measured_bytes.saturating_add(size);
1091 targets.push((state.output_activity(), id.clone(), path, size));
1092 }
1093 }
1094 if under_pressure {
1095 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1098 } else {
1099 targets.sort_by_key(|t| t.0);
1100 }
1101 let free_before = crate::disk::free_bytes(home).ok();
1102 #[cfg(unix)]
1103 {
1104 let mut seen = std::collections::BTreeSet::new();
1105 report.measured_allocated_file_bytes = Some(0);
1106 report.estimated_released_allocated_bytes = Some(0);
1107 report.proven_reclaimable_allocated_bytes = Some(0);
1108 for (_, _, path, _) in &targets {
1109 report.measured_allocated_file_bytes = report
1110 .measured_allocated_file_bytes
1111 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1112 .map(|(a, b)| a.saturating_add(b));
1113 }
1114 }
1115 let mut remaining = report.measured_bytes;
1116 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1117 let mut deletion_started = false;
1118 let mut oversized_started = false;
1119 for (updated, id, path, _) in targets {
1120 if disk.worktree_limit_bytes == 0
1121 || remaining <= disk.worktree_limit_bytes
1122 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1123 {
1124 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1125 continue;
1126 }
1127 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1128 report
1129 .retained
1130 .push(format!("{id}: per-pass byte budget spent"));
1131 continue;
1132 }
1133 let result = async {
1134 ensure!(
1135 crate::clean::due(now, updated, disk.fold_grace_secs),
1136 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1137 );
1138 let _owner = crate::pr_head::owner_lock(home, &id)?;
1139 let state = RunState::load_under(&id, home)?;
1140 ensure!(
1141 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1142 == Liveness::Dead,
1143 "run live or ownership unknown"
1144 );
1145 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1146 let _driver = crate::driver::maintenance(home, &id)?;
1147 let _outputs = OutputLease::take(&path, true)?;
1148 ensure!(
1149 worktrees(&state).await?.contains(&path),
1150 "worktree registration changed"
1151 );
1152 let target = validate_cargo_target(&path, true).await
1153 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1154 let _cargo = cargo_locks(&target)
1155 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1156 validate_cargo_target(&path, true).await
1159 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1160 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1161 if lease_meta.len() > 0 {
1162 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1163 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1164 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1165 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1166 }
1167 output_age(&target, now, disk.fold_grace_secs)
1168 .with_context(|| format!("stage output-age: {}", target.display()))?;
1169 let mut outputs = declared_outputs(&target)
1170 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1171 let incremental = match crate::incremental::candidates(&path) {
1172 Ok(candidates) => candidates,
1173 Err(error) => {
1174 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1175 crate::incremental::Reclaim::default()
1176 }
1177 };
1178 let incremental_files = incremental.files.clone();
1179 outputs.extend(incremental_files.iter().cloned());
1180 outputs.sort();
1181 outputs.dedup();
1182 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1183 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1184 .with_context(|| format!("stage measure: {}", p.display())))
1185 .collect::<Result<Vec<_>>>()?;
1186 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1187 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1188 let mut allocation = ReleasedAllocation::of(&outputs)?;
1189 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1190 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1191 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1192 ensure!(!deletion_started || !exceeds_budget,
1196 "declared outputs exceed remaining per-pass byte budget");
1197 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1200 std::fs::write(
1201 &ledger,
1202 serde_json::to_vec(
1203 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1204 )?,
1205 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1206 if !incremental_files.is_empty() {
1209 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1210 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1211 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1212 }
1213 ensure!(worktrees(&state).await?.contains(&path),
1214 "worktree registration changed before deletion");
1215 deletion_started = true;
1216 oversized_started = exceeds_budget;
1217 measurements.remove(&target);
1218 for (output, size) in outputs.iter().zip(sizes) {
1219 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1220 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1221 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1222 report.freed_bytes = report.freed_bytes.saturating_add(size);
1224 remaining = remaining.saturating_sub(size);
1225 }
1226 incremental.finish().context("finish incremental reclamation")?;
1227 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1228 Ok::<_, anyhow::Error>(())
1229 }
1230 .await;
1231 if let Err(e) = result {
1232 report.retained.push(format!(
1233 "{id}/{}: {e:#}",
1234 path.file_name().unwrap_or_default().to_string_lossy()
1235 ));
1236 }
1237 }
1238 measurements.retain(|path, _| observed.contains(path));
1239 if let Ok(body) = serde_json::to_vec(&measurements)
1240 && let Err(e) = std::fs::write(&measurements_path, body)
1241 {
1242 report.failures.push(format!("output measurements: {e}"));
1243 }
1244 copies::evict(home, disk, now, &mut report).await;
1245 retire_empty(home, disk, now, &mut report).await;
1246 report.protected_or_unproven_bytes = report
1247 .measured_bytes
1248 .saturating_sub(report.proven_reclaimable_bytes);
1249 let pressure_remaining = pressure_paths
1250 .iter()
1251 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1252 if !report.failures.is_empty()
1253 || disk.worktree_limit_bytes > 0
1254 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1255 {
1256 let details = report
1257 .retained
1258 .iter()
1259 .filter(|reason| {
1260 !reason.contains("within policy")
1261 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1262 })
1263 .take(4)
1264 .chain(report.failures.iter().take(2))
1265 .cloned()
1266 .collect::<Vec<_>>()
1267 .join("; ");
1268 crate::notices::raise_in(
1269 home,
1270 crate::notices::Notice::warn(
1271 "disk:retention",
1272 format!(
1273 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1274 report.measured_bytes,
1275 report.proven_reclaimable_bytes,
1276 report.protected_or_unproven_bytes,
1277 report.freed_bytes,
1278 if report.proven_reclaimable_bytes == 0 {
1279 "No inspected contents have deletion authority."
1280 } else {
1281 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1282 }
1283 ),
1284 ),
1285 );
1286 }
1287 report.observed_home_free_change_bytes = free_before
1288 .zip(crate::disk::free_bytes(home).ok())
1289 .map(|(before, after)| {
1290 (i128::from(after) - i128::from(before))
1291 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1292 });
1293 let store = home.join("retention.json");
1294 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1295 let tmp = home.join("retention.json.tmp");
1296 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1297 tracing::warn!("retention report could not be saved: {e}");
1298 }
1299 }
1300 tracing::info!(
1301 measured_bytes = report.measured_bytes,
1302 freed_bytes = report.freed_bytes,
1303 retired = report.retired,
1304 retained = report.retained.len(),
1305 failures = report.failures.len(),
1306 "retained worktree cleanup"
1307 );
1308 for reason in report.retained.iter().chain(&report.failures) {
1309 tracing::debug!(%reason, "worktree retained");
1310 }
1311 report
1312}
1313
1314async fn retire_empty(
1315 home: &Path,
1316 disk: &crate::config::Disk,
1317 now: Timestamp,
1318 report: &mut Report,
1319) {
1320 if !disk.auto_fold {
1321 return;
1322 }
1323 let queue = crate::queue::Queue::at(home.join("queue"));
1324 let questions = crate::ask::Questions::at(home.join("questions"));
1325 let open = questions.list();
1326 for listed in queue.list() {
1327 let Ok(_claim) = queue.claim(&listed.id) else {
1328 continue;
1329 };
1330 let Ok(task) = queue.get(&listed.id) else {
1331 continue;
1332 };
1333 if open
1334 .iter()
1335 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1336 {
1337 report.retained.push(format!(
1338 "{}: task has an unanswered owner decision",
1339 task.id
1340 ));
1341 continue;
1342 }
1343 let sources: Vec<_> = task
1344 .runs
1345 .iter()
1346 .collect::<std::collections::BTreeSet<_>>()
1347 .into_iter()
1348 .cloned()
1349 .collect();
1350 let mut useful = Vec::new();
1351 let mut selected = Vec::new();
1352 let mut public = Vec::new();
1353 let mut unreadable = false;
1354 for id in &sources {
1355 let Ok(state) = RunState::load_under(id, home) else {
1356 unreadable = true;
1357 continue;
1358 };
1359 if state.released() || state.retired_to.is_some() {
1360 continue;
1361 }
1362 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1363 selected.push(state.clone());
1364 }
1365 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1366 public.push(state.clone());
1367 }
1368 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1369 useful.push(state);
1370 }
1371 }
1372 let pool = if !selected.is_empty() {
1376 selected
1377 } else if !public.is_empty() {
1378 public
1379 } else {
1380 useful
1381 };
1382 if unreadable || pool.len() != 1 {
1383 report.retained.push(format!(
1384 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1385 task.id,
1386 sources.len()
1387 ));
1388 continue;
1389 }
1390 let authority = &pool[0];
1391 if authority.candidates.is_empty() {
1392 continue;
1393 }
1394 let latest = &authority.id;
1395 for id in &sources {
1396 if id == latest || report.retired >= 5 {
1397 continue;
1398 }
1399 if let Ok(state) = RunState::load_under(id, home)
1400 && state.retired_to.as_ref() == Some(latest)
1401 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1402 {
1403 continue;
1404 }
1405 let result = async {
1406 ensure!(
1407 !task
1408 .resume_override
1409 .as_ref()
1410 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1411 "operator-pinned recovery run"
1412 );
1413 ensure!(
1414 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1415 "selected replacement recovery source"
1416 );
1417 let _owner = crate::pr_head::owner_lock(home, id)?;
1418 let mut state = RunState::load_under(id, home)?;
1419 ensure!(
1420 state.repo == authority.repo,
1421 "repository differs from recovery authority"
1422 );
1423 ensure!(
1424 !crate::pr_recovery::explicit_sources_retained(home, id),
1425 "explicitly retained recovery source"
1426 );
1427 ensure!(
1428 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1429 == Liveness::Dead,
1430 "run live or ownership unknown"
1431 );
1432 let _driver = crate::driver::maintenance(home, id)?;
1433 ensure!(
1434 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1435 "within grace period"
1436 );
1437 ensure!(
1438 !open.iter().any(|q| q.run == *id && q.status.open()),
1439 "unanswered owner decision"
1440 );
1441 ensure!(
1442 state.pr.is_none()
1443 && state.pr_head.is_none()
1444 && state.pr_recovery.is_none()
1445 && state.land_approval.is_none()
1446 && state.merge_decision.is_none()
1447 && state.reviews.is_empty()
1448 && state.recovery.is_none()
1449 && state.candidate_preservation.is_empty()
1450 && state.interrupted_implementations.is_empty()
1451 && !state.released(),
1452 "PR ownership or recovery checkpoint"
1453 );
1454 let proof = prove_empty(home, &state, latest).await?;
1455 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1456 for path in &proof.paths {
1459 ensure!(
1460 git::worktree_remove_clean(&state.repo, path).await?,
1461 "Git refused clean worktree removal"
1462 );
1463 }
1464 Ok::<_, anyhow::Error>(())
1465 }
1466 .await;
1467 match result {
1468 Ok(()) => report.retired += 1,
1469 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1470 }
1471 }
1472 }
1473}
1474
1475struct EmptyProof {
1476 paths: Vec<PathBuf>,
1477 _outputs: Vec<OutputLease>,
1478 _cargo: Vec<File>,
1479}
1480
1481pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1484 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1485 let mut files = Vec::new();
1486 let mut blobs = Vec::new();
1487 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1488 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1489 let parts: Vec<_> = header.split_whitespace().collect();
1490 ensure!(
1491 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1492 "unsupported tracked source kind"
1493 );
1494 let path = worktree.join(name);
1495 let meta = std::fs::symlink_metadata(&path)?;
1496 ensure!(
1497 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1498 "noncanonical tracked source"
1499 );
1500 #[cfg(unix)]
1501 {
1502 use std::os::unix::fs::PermissionsExt as _;
1503 ensure!(
1504 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1505 "tracked executable mode differs from original base"
1506 );
1507 }
1508 path.to_str().context("non UTF-8 source")?;
1509 files.push(git::path_arg(&path));
1510 blobs.push(parts[2].to_owned());
1511 }
1512 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1513 let mut args = vec!["hash-object", "--no-filters", "--"];
1514 args.extend(paths.iter().map(String::as_str));
1515 let actual = git::git(worktree, &args).await?;
1516 ensure!(
1517 actual.lines().eq(expected.iter().map(String::as_str)),
1518 "tracked bytes differ from original base"
1519 );
1520 }
1521 Ok(())
1522}
1523
1524async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1525 let id = &state.id;
1526 let paths = worktrees(state).await?;
1527 let mut guards = Vec::new();
1528 let mut cargo_guards = Vec::new();
1529 for path in &paths {
1530 guards.push(OutputLease::take(path, true)?);
1531 ensure!(
1532 git::rev_parse(path, "HEAD").await? == state.base_commit,
1533 "unique or unrecorded commits"
1534 );
1535 ensure!(
1536 git::git(
1537 path,
1538 &[
1539 "--no-optional-locks",
1540 "status",
1541 "--porcelain",
1542 "--untracked-files=all"
1543 ]
1544 )
1545 .await?
1546 .is_empty(),
1547 "tracked or untracked source changes"
1548 );
1549 ensure!(
1550 git::git(
1551 path,
1552 &[
1553 "--no-optional-locks",
1554 "status",
1555 "--porcelain",
1556 "--untracked-files=all",
1557 "--ignored",
1558 "--",
1559 ".",
1560 ":(exclude)target"
1561 ]
1562 )
1563 .await?
1564 .is_empty(),
1565 "source changes, ignored files or build outputs"
1566 );
1567 tracked_base(path, &state.base_commit).await?;
1568 match std::fs::symlink_metadata(path.join("target")) {
1569 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1570 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1571 Err(error) => return Err(error.into()),
1572 }
1573 for checkpoint in [
1574 "rebase-merge",
1575 "rebase-apply",
1576 "sequencer",
1577 "BISECT_LOG",
1578 "REVERT_HEAD",
1579 "AUTO_MERGE",
1580 "MERGE_MSG",
1581 "CHERRY_PICK_HEAD",
1582 "MERGE_HEAD",
1583 "REBASE_HEAD",
1584 ] {
1585 let location = git::git(
1586 path,
1587 &[
1588 "rev-parse",
1589 "--path-format=absolute",
1590 "--git-path",
1591 checkpoint,
1592 ],
1593 )
1594 .await?;
1595 ensure!(
1596 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1597 "retained Git recovery metadata: {checkpoint}"
1598 );
1599 }
1600 let origin_path = git::git(
1603 path,
1604 &[
1605 "rev-parse",
1606 "--path-format=absolute",
1607 "--git-path",
1608 "ORIG_HEAD",
1609 ],
1610 )
1611 .await?;
1612 match std::fs::symlink_metadata(origin_path.trim()) {
1613 Ok(meta) => {
1614 ensure!(
1615 meta.is_file() && !meta.file_type().is_symlink(),
1616 "unknown ORIG_HEAD artifact"
1617 );
1618 git::rev_parse(path, "ORIG_HEAD")
1619 .await
1620 .context("unreadable ORIG_HEAD")?;
1621 }
1622 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1623 Err(e) => return Err(e.into()),
1624 }
1625 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1626 ensure!(
1627 git::git_raw(
1628 path,
1629 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1630 )
1631 .await?
1632 .ok(),
1633 "ORIG_HEAD retains commits outside the original base"
1634 );
1635 }
1636 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1637 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1638 ensure!(
1639 git::git_raw(
1640 path,
1641 &["merge-base", "--is-ancestor", head, &state.base_commit]
1642 )
1643 .await?
1644 .ok(),
1645 "worktree reflog retains commits outside the original base"
1646 );
1647 }
1648 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1649 ensure!(
1650 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1651 .await?
1652 .ok(),
1653 "unfinished Git operation"
1654 );
1655 }
1656 }
1657 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1660 return Ok(EmptyProof {
1661 paths,
1662 _outputs: guards,
1663 _cargo: cargo_guards,
1664 });
1665 }
1666 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1667 for candidate in &state.candidates {
1668 ensure!(
1669 (paths.contains(&candidate.worktree)
1670 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1671 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1672 "candidate recovery source"
1673 );
1674 }
1675 let pattern = format!("refs/heads/magi/{}/", state.short());
1676 let branches = git::git(
1677 &state.repo,
1678 &["for-each-ref", "--format=%(objectname)", &pattern],
1679 )
1680 .await?;
1681 ensure!(
1682 branches.lines().all(|head| head == state.base_commit),
1683 "unrecorded candidate branch changes"
1684 );
1685 Ok(EmptyProof {
1686 paths,
1687 _outputs: guards,
1688 _cargo: cargo_guards,
1689 })
1690}
1691
1692fn commit_retirement(
1697 home: &Path,
1698 queue: &crate::queue::Queue,
1699 expected: &crate::queue::Task,
1700 state: &mut RunState,
1701 authority: &str,
1702 sources: &[String],
1703 now: Timestamp,
1704) -> Result<()> {
1705 let _write = queue.lock_task(&expected.id)?;
1706 let current = queue.get(&expected.id)?;
1707 ensure!(
1708 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1709 "task changed during retirement proof; retaining worktrees for the current owner decision"
1710 );
1711 ensure!(
1712 !current
1713 .resume_override
1714 .as_ref()
1715 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1716 && !current
1717 .replacement
1718 .as_ref()
1719 .is_some_and(|pin| pin.run == state.id),
1720 "current owner selected this recovery source"
1721 );
1722 ensure!(
1723 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1724 "explicitly retained recovery source"
1725 );
1726 ensure!(
1727 !crate::ask::Questions::at(home.join("questions"))
1728 .list()
1729 .iter()
1730 .any(|q| q.status.open()
1731 && (q.run == state.id
1732 || q.run == current.id
1733 || current.blocked_by.contains(&q.id))),
1734 "owner decision arrived during retirement proof"
1735 );
1736 let dir = home.join("recovery-lineages");
1737 std::fs::create_dir_all(&dir)?;
1738 let tmp = dir.join(format!("{}.tmp", current.id));
1739 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1740 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1741 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1742 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1743 state.retired_to = Some(authority.to_owned());
1744 state.event(
1745 "retention",
1746 format!(
1747 "empty attempt retired to {authority}; recovery sources: {}",
1748 sources.join(", ")
1749 ),
1750 );
1751 state.save_under(home)?;
1752 Ok(())
1754}
1755
1756pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1760 if min_free == 0 {
1761 return std::future::pending().await;
1762 }
1763 loop {
1764 for path in &paths {
1765 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1766 match crate::disk::free_bytes(existing) {
1767 Ok(free) if free >= min_free / 2 => {}
1768 Ok(free) => {
1769 return format!(
1770 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1771 );
1772 }
1773 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1774 }
1775 }
1776 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1777 }
1778}
1779
1780#[cfg(all(test, unix))]
1781mod tests {
1782 use super::*;
1783 use crate::{
1784 config::{Config, Disk},
1785 queue::{Source, Task},
1786 run::{Candidate, RunStatus},
1787 };
1788
1789 struct Fx {
1790 _temp: tempfile::TempDir,
1791 home: PathBuf,
1792 repo: PathBuf,
1793 bay: PathBuf,
1794 base: String,
1795 }
1796 impl Fx {
1797 async fn new() -> Self {
1798 let temp = tempfile::tempdir().unwrap();
1799 let root = temp.path().canonicalize().unwrap();
1800 let home = root.join("home");
1801 let repo = root.join("repo");
1802 let bay = root.join("worktrees");
1803 std::fs::create_dir_all(&repo).unwrap();
1804 std::fs::create_dir_all(home.join("runs")).unwrap();
1805 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1806 git::git(&repo, &["config", "user.name", "Fixture"])
1807 .await
1808 .unwrap();
1809 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1810 .await
1811 .unwrap();
1812 git::git(&repo, &["config", "commit.gpgsign", "false"])
1813 .await
1814 .unwrap();
1815 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1816 std::fs::write(repo.join("source"), "base\n").unwrap();
1817 git::git(&repo, &["add", "."]).await.unwrap();
1818 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1819 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1820 Self {
1821 _temp: temp,
1822 home,
1823 repo,
1824 bay,
1825 base,
1826 }
1827 }
1828 async fn run(&self, tag: &str, useful: bool) -> RunState {
1829 let mut cfg = Config::default();
1830 cfg.graph.worktree_root = Some(self.bay.clone());
1831 let mut state = RunState::new(
1832 self.repo.clone(),
1833 "main".into(),
1834 self.base.clone(),
1835 "task".into(),
1836 cfg,
1837 );
1838 state.id = format!("20260101-000000-{tag}");
1839 state.status = RunStatus::Blocked;
1840 state.driver_exited = true;
1841 let wt = state.worktree_root().join("cand-A");
1842 let branch = format!("magi/{tag}/A");
1843 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1844 .await
1845 .unwrap();
1846 if useful {
1847 std::fs::write(wt.join("source"), tag).unwrap();
1848 git::git(&wt, &["add", "."]).await.unwrap();
1849 git::git(&wt, &["commit", "-m", "candidate change"])
1850 .await
1851 .unwrap();
1852 }
1853 state.candidates.push(Candidate {
1854 index: 0,
1855 label: 'A',
1856 agent: "fixture".into(),
1857 branch,
1858 worktree: wt,
1859 summary: String::new(),
1860 stat: String::new(),
1861 files: usize::from(useful),
1862 commits: usize::from(useful),
1863 empty: !useful,
1864 failed: None,
1865 verified_noop: None,
1866 duration_ms: 0,
1867 folded: false,
1868 });
1869 state.save_under(&self.home).unwrap();
1870 state
1871 }
1872 fn disk() -> Disk {
1873 Disk {
1874 worktree_limit_bytes: 1,
1875 worktree_reclaim_bytes: 1024 * 1024,
1876 fold_grace_secs: 0,
1877 min_free_bytes: 0,
1878 ..Disk::default()
1879 }
1880 }
1881 fn target(wt: &Path) {
1882 let target = wt.join("target");
1883 std::fs::create_dir_all(target.join("debug")).unwrap();
1884 let mut artifact = [1; 200];
1885 artifact[..4].copy_from_slice(b"\x7fELF");
1886 std::fs::write(target.join("debug/output"), artifact).unwrap();
1887 let output = target
1888 .join("debug/output")
1889 .display()
1890 .to_string()
1891 .replace('\\', "\\\\")
1892 .replace(' ', "\\ ");
1893 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1894 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1895 std::fs::write(
1896 target.join("CACHEDIR.TAG"),
1897 "Signature: 8a477f597d28d172789f06886806bc55\n",
1898 )
1899 .unwrap();
1900 std::fs::write(
1901 target.join(".rustc_info.json"),
1902 "{\"rustc_fingerprint\":123}",
1903 )
1904 .unwrap();
1905 }
1906 fn now() -> Timestamp {
1907 Timestamp::now() + jiff::SignedDuration::from_secs(2)
1908 }
1909 fn task(&self, runs: &[RunState]) -> Task {
1910 let mut task = Task::new(
1911 "fixture".into(),
1912 "task".into(),
1913 self.repo.clone(),
1914 Source::Human,
1915 );
1916 task.runs = runs.iter().map(|s| s.id.clone()).collect();
1917 crate::queue::Queue::at(self.home.join("queue"))
1918 .put(&mut task)
1919 .unwrap();
1920 task
1921 }
1922 }
1923
1924 #[cfg(unix)]
1925 #[tokio::test]
1926 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1927 let f = Fx::new().await;
1928 let state = f.run("ca01", true).await;
1929 let wt = &state.candidates[0].worktree;
1930 let generation = crate::incremental::tests::generate(wt);
1931 for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1933 for entry in std::fs::read_dir(dir).unwrap() {
1934 let path = entry.unwrap().path();
1935 if path.extension().is_some_and(|x| x == "d") {
1936 std::fs::remove_file(path).unwrap();
1937 }
1938 }
1939 }
1940 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1941 assert_eq!(
1942 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1943 0,
1944 "live generation is protected"
1945 );
1946 drop(generation);
1947 let candidates = crate::incremental::candidates(wt).unwrap();
1948 let logical: u64 = candidates
1949 .files
1950 .iter()
1951 .map(|p| std::fs::metadata(p).unwrap().len())
1952 .sum();
1953 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1954 std::fs::write(&unknown, "retain unknown data").unwrap();
1955 drop(candidates);
1956 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1957 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1958 assert!(
1959 refused
1960 .retained
1961 .iter()
1962 .any(|s| s.contains("provenance mismatch"))
1963 );
1964 std::fs::remove_file(&unknown).unwrap();
1965 let legacy = wt.join("target/debug/incremental/legacy-session");
1966 std::fs::create_dir(&legacy).unwrap();
1967 let candidates = crate::incremental::candidates(wt).unwrap();
1969 let bin = candidates
1970 .files
1971 .iter()
1972 .find(|p| p.extension().is_some_and(|x| x == "bin"))
1973 .unwrap();
1974 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1975 drop(candidates);
1976 let cargo = cargo_locks(&wt.join("target")).unwrap();
1977 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1978 drop(cargo);
1979 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1980 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1981 drop(outputs);
1982 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1983 assert_eq!(report.freed_bytes, logical, "{report:?}");
1984 assert_eq!(report.proven_reclaimable_bytes, logical);
1985 assert!(report.measured_bytes > logical);
1986 assert_eq!(
1987 report.protected_or_unproven_bytes,
1988 report.measured_bytes - logical
1989 );
1990 #[cfg(unix)]
1991 {
1992 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
1993 assert!(
1994 report.measured_allocated_file_bytes.unwrap()
1995 >= report.estimated_released_allocated_bytes.unwrap()
1996 );
1997 }
1998 assert!(report.observed_home_free_change_bytes.is_some());
1999 assert!(legacy.join("query-cache.bin").is_file());
2000 assert!(wt.join("src/lib.rs").is_file());
2001 assert_eq!(report.retired, 0);
2002 }
2003
2004 #[cfg(unix)]
2005 #[tokio::test]
2006 async fn incremental_and_deps_links_account_for_retained_third_link() {
2007 use std::os::unix::fs::MetadataExt;
2008 let f = Fx::new().await;
2009 let state = f.run("ca02", true).await;
2010 let wt = &state.candidates[0].worktree;
2011 let generation = crate::incremental::tests::generate(wt);
2012 drop(generation);
2013 let candidates = crate::incremental::candidates(wt).unwrap();
2014 let objects = candidates
2015 .files
2016 .iter()
2017 .filter(|p| p.extension().is_some_and(|x| x == "o"))
2018 .take(2)
2019 .cloned()
2020 .collect::<Vec<_>>();
2021 assert_eq!(
2022 objects.len(),
2023 2,
2024 "fixture must produce real incremental objects"
2025 );
2026 let mut outputs = candidates.files.clone();
2027 drop(candidates);
2028 let target = wt.join("target");
2029 for dir in [target.join("debug"), target.join("debug/deps")] {
2030 for entry in std::fs::read_dir(dir).unwrap() {
2031 let path = entry.unwrap().path();
2032 if path.extension().is_some_and(|x| x == "d") {
2033 std::fs::remove_file(path).unwrap();
2034 }
2035 }
2036 }
2037 for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
2040 let path = entry.unwrap().path();
2041 if path.is_file() {
2042 std::fs::remove_file(path).unwrap();
2043 }
2044 }
2045 for (index, object) in objects.iter().enumerate() {
2046 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2047 let dep = target.join(format!("debug/deps/link{index}"));
2048 std::fs::hard_link(object, &dep).unwrap();
2049 std::fs::write(
2050 dep.with_extension("d"),
2051 format!("{}: src/lib.rs\n", dep.display()),
2052 )
2053 .unwrap();
2054 outputs.push(dep);
2055 }
2056 let retained = wt.join("retained-input");
2057 std::fs::hard_link(&objects[1], &retained).unwrap();
2058 let sentinel = std::fs::read(&retained).unwrap();
2059 let logical: u64 = outputs
2060 .iter()
2061 .map(|p| std::fs::metadata(p).unwrap().len())
2062 .sum();
2063 let mut seen = std::collections::BTreeSet::new();
2064 let retained_meta = std::fs::metadata(&retained).unwrap();
2065 let retained_inode = (retained_meta.dev(), retained_meta.ino());
2066 let mut expected = 0u64;
2067 for output in &outputs {
2068 let m = std::fs::metadata(output).unwrap();
2069 let inode = (m.dev(), m.ino());
2070 if inode != retained_inode && seen.insert(inode) {
2071 expected += m.blocks() * 512;
2072 }
2073 }
2074 assert!(expected > 0);
2075 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2076 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2077 assert_eq!(report.freed_bytes, logical, "{report:?}");
2078 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2079 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2080 assert!(outputs.iter().all(|p| !p.exists()));
2081 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2082 assert!(wt.join("src/lib.rs").is_file());
2083 }
2084
2085 #[tokio::test]
2086 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2087 let f = Fx::new().await;
2088 let state = f.run("ca03", true).await;
2089 let wt = &state.candidates[0].worktree;
2090 Fx::target(wt);
2091 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2092 let unknown = wt.join("target/debug/operator-note");
2093 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2094 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2095 assert_eq!(report.proven_reclaimable_bytes, 0);
2096 #[cfg(unix)]
2097 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2098 assert!(report.protected_or_unproven_bytes >= 4096);
2099 let message = || {
2100 crate::notices::Notices::at(f.home.join("notifications"))
2101 .list()
2102 .into_iter()
2103 .find(|n| n.key == "disk:retention")
2104 .unwrap()
2105 .message
2106 };
2107 let notice = message();
2108 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2109 assert!(notice.contains("protected or unproven"), "{notice}");
2110 assert!(
2111 notice.contains("legacy incremental/build and unknown files retained"),
2112 "{notice}"
2113 );
2114 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2115 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2116 assert_eq!(report.freed_bytes, 0);
2117 let notice = message();
2118 assert!(notice.contains(&state.id), "{notice}");
2119 assert!(
2120 notice.contains("lock") || notice.contains("owner"),
2121 "{notice}"
2122 );
2123 drop(owner);
2124 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2125 }
2126
2127 #[tokio::test]
2128 async fn real_cargo_build_holds_the_target_against_reclamation() {
2129 let f = Fx::new().await;
2130 let state = f.run("ca02", true).await;
2131 let wt = &state.candidates[0].worktree;
2132 Fx::target(wt);
2133 std::fs::create_dir(wt.join("src")).unwrap();
2134 std::fs::write(
2135 wt.join("Cargo.toml"),
2136 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2137 )
2138 .unwrap();
2139 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2140 std::fs::write(
2141 wt.join("build.rs"),
2142 r#"fn main() {
2143 let ready = std::env::var("FIXTURE_READY").unwrap();
2144 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2145 std::fs::write(ready, "ready").unwrap();
2146 let start = std::time::Instant::now();
2147 while !std::path::Path::new(&release).exists() && start.elapsed().as_secs() < 15 {
2148 std::thread::sleep(std::time::Duration::from_millis(10));
2149 }
2150 }"#,
2151 )
2152 .unwrap();
2153 let ready = wt.join("ready");
2154 let release = wt.join("release");
2155 let mut cargo = tokio::process::Command::new("cargo");
2156 cargo.env_clear();
2157 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2158 if let Some(value) = std::env::var_os(key) {
2159 cargo.env(key, value);
2160 }
2161 }
2162 let mut child = cargo
2163 .current_dir(wt)
2164 .env("RUSTUP_TOOLCHAIN", "stable")
2165 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2166 .env("CARGO_INCREMENTAL", "1")
2167 .env("FIXTURE_READY", &ready)
2168 .env("FIXTURE_RELEASE", &release)
2169 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2170 .arg(wt.join("target"))
2171 .stdout(std::process::Stdio::null())
2172 .stderr(std::process::Stdio::null())
2173 .kill_on_drop(true)
2174 .spawn()
2175 .unwrap();
2176 let arrived = tokio::time::timeout(std::time::Duration::from_secs(10), async {
2177 while !ready.exists() {
2178 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2179 }
2180 })
2181 .await
2182 .is_ok();
2183 let report = if arrived {
2184 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2185 } else {
2186 None
2187 };
2188 std::fs::write(release, "release").unwrap();
2190 let status = tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await;
2191 if status.is_err() {
2192 child.kill().await.unwrap();
2193 child.wait().await.unwrap();
2194 }
2195 assert!(
2196 arrived,
2197 "fixture build never reached its active build script"
2198 );
2199 assert!(status.unwrap().unwrap().success());
2200 let report = report.unwrap();
2201 assert_eq!(report.freed_bytes, 0, "{report:?}");
2202 assert!(
2203 report
2204 .retained
2205 .iter()
2206 .any(|s| s.contains("Cargo build is active")),
2207 "{report:?}"
2208 );
2209 assert!(wt.join("target/debug/output").is_file());
2210 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2211 }
2212
2213 #[cfg(unix)]
2214 #[test]
2215 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2216 let temp = tempfile::tempdir().unwrap();
2217 let a = temp.path().join("a");
2218 let b = temp.path().join("b");
2219 let outside = temp.path().join("retained-input");
2220 std::fs::write(&a, vec![1; 65536]).unwrap();
2221 std::fs::hard_link(&a, &b).unwrap();
2222 std::fs::hard_link(&a, &outside).unwrap();
2223 let mut seen = Default::default();
2224 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2225 assert!(blocks > 0);
2226 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2227 std::fs::remove_file(&a).unwrap();
2228 assert_eq!(accounting.unlinked(&a), Some(0));
2229 std::fs::remove_file(&b).unwrap();
2230 assert_eq!(accounting.unlinked(&b), Some(0));
2231 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2232 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2233 std::fs::remove_file(&outside).unwrap();
2234 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2235 }
2236
2237 #[cfg(unix)]
2238 #[test]
2239 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2240 let temp = tempfile::tempdir_in(".").unwrap();
2241 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2242 assert!(home.is_relative());
2243 let paths = vec![std::env::current_dir().unwrap()];
2244 let gib = 1024 * 1024 * 1024;
2245 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2246 assert!(
2247 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2248 9 * gib
2249 ))
2250 .is_err()
2251 );
2252 drop(reservation);
2253 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2254 let bad = temp.path().join("symlink-state");
2255 std::fs::create_dir(&bad).unwrap();
2256 std::os::unix::fs::symlink(
2257 home.canonicalize().unwrap().join("disk-admission"),
2258 bad.join("disk-admission"),
2259 )
2260 .unwrap();
2261 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2262 }
2263
2264 #[cfg(unix)]
2265 #[test]
2266 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2267 let temp = tempfile::tempdir().unwrap();
2268 let home = temp.path().canonicalize().unwrap();
2269 let paths = vec![home.clone(), home.join("not-created-yet")];
2270 let gib = 1024 * 1024 * 1024;
2271 let floor = 8 * gib;
2272 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2273 assert_eq!(first.len(), 1);
2274 let blocked =
2275 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2276 assert!(blocked.to_string().contains("active writer reservations"));
2277 let second =
2278 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2279 drop(first);
2280 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2281 drop(second);
2282 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2283 assert!(
2284 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2285 );
2286 }
2287
2288 #[cfg(unix)]
2289 #[test]
2290 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2291 let temp = tempfile::tempdir().unwrap();
2292 let home = temp.path().canonicalize().unwrap();
2293 let barrier = std::sync::Barrier::new(2);
2294 let gib = 1024 * 1024 * 1024;
2295 let results = std::thread::scope(|scope| {
2296 let threads: Vec<_> = (0..2)
2297 .map(|_| {
2298 scope.spawn(|| {
2299 barrier.wait();
2300 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2301 Ok(100 + gib)
2302 })
2303 })
2304 })
2305 .collect();
2306 threads
2307 .into_iter()
2308 .map(|thread| thread.join().unwrap())
2309 .collect::<Vec<_>>()
2310 });
2311 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2312 assert!(
2313 results
2314 .iter()
2315 .filter_map(|r| r.as_ref().err())
2316 .all(|e| e.to_string().contains("active writer reservations"))
2317 );
2318 }
2319
2320 #[tokio::test]
2321 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2322 real_cargo_fixture(false).await;
2323 }
2324
2325 #[tokio::test]
2326 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2327 real_cargo_fixture(true).await;
2328 }
2329
2330 async fn real_cargo_fixture(precreate: bool) {
2331 let f = Fx::new().await;
2332 let state = f.run("ca00", true).await;
2333 let wt = &state.candidates[0].worktree;
2334 let source = "fn main() { println!(\"retained fixture\"); }\n";
2335 std::fs::create_dir(wt.join("src")).unwrap();
2336 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2337 std::fs::write(
2338 wt.join("Cargo.toml"),
2339 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2340 )
2341 .unwrap();
2342 if precreate {
2343 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2344 }
2345 let mut cargo = std::process::Command::new("cargo");
2346 cargo.env_clear();
2347 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2348 if let Some(value) = std::env::var_os(key) {
2349 cargo.env(key, value);
2350 }
2351 }
2352 let build = cargo
2353 .current_dir(wt)
2354 .env("RUSTUP_TOOLCHAIN", "stable")
2355 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2356 .env("CARGO_INCREMENTAL", "0")
2357 .args([
2358 "build",
2359 "--offline",
2360 "--release",
2361 "--jobs",
2362 "1",
2363 "--target-dir",
2364 ])
2365 .arg(wt.join("target"))
2366 .output()
2367 .unwrap();
2368 assert!(
2369 build.status.success(),
2370 "{}",
2371 String::from_utf8_lossy(&build.stderr)
2372 );
2373 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2374 if precreate {
2375 let again = cargo.output().unwrap();
2377 assert!(
2378 again.status.success(),
2379 "{}",
2380 String::from_utf8_lossy(&again.stderr)
2381 );
2382 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2383 assert!(cargo_target(wt).await.is_err());
2384 }
2385 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2386 .await
2387 .unwrap();
2388 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2389 .await
2390 .unwrap();
2391 let artifact = wt.join("target/release/retained-fixture");
2392 assert!(artifact.is_file());
2393 let unknown = wt.join("target/release/operator-note");
2394 std::fs::write(&unknown, "keep this note\n").unwrap();
2395
2396 let target = wt.join("target");
2397 let mut preserved = std::collections::BTreeMap::new();
2398 for relative in [
2399 "tmp/sentinel",
2400 "magi-tools/sentinel",
2401 "lifecycle-client-check.log",
2402 ] {
2403 let path = target.join(relative);
2404 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2405 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2406 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2407 }
2408 let decoy = target.join("magi-tools/retained-fixture");
2411 std::fs::copy(&artifact, &decoy).unwrap();
2412 let dep_info = decoy.with_extension("d");
2413 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2414 for path in [
2415 decoy,
2416 dep_info,
2417 wt.join("Cargo.toml"),
2418 wt.join("Cargo.lock"),
2419 ] {
2420 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2421 }
2422 let proven = declared_outputs(&target).unwrap();
2423
2424 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2425 assert!(out.freed_bytes > 0, "{out:?}");
2426 for (path, bytes) in preserved {
2427 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2428 }
2429 assert!(!artifact.exists(), "{out:?}");
2430 assert_eq!(
2431 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2432 source
2433 );
2434 assert_eq!(
2435 std::fs::read_to_string(&unknown).unwrap(),
2436 "keep this note\n"
2437 );
2438 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2439 let ledger: serde_json::Value = serde_json::from_slice(
2440 &std::fs::read(
2441 f.home
2442 .join("runs")
2443 .join(&state.id)
2444 .join("build-outputs.json"),
2445 )
2446 .unwrap(),
2447 )
2448 .unwrap();
2449 assert_eq!(
2450 ledger["identity"],
2451 if precreate {
2452 "rustc-profile-dep-info"
2453 } else {
2454 "cache-tag"
2455 }
2456 );
2457 if precreate {
2458 assert!(!target.join("CACHEDIR.TAG").exists());
2459 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2460 }
2461 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2462 assert!(deleted.contains(&artifact));
2463 assert_eq!(deleted, proven);
2464 for path in deleted {
2465 assert!(!path.exists());
2466 assert!(
2467 [target.join("release"), target.join("release/deps")]
2468 .contains(&path.parent().unwrap().to_owned())
2469 );
2470 }
2471 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2472 }
2473
2474 fn missing_tag_target(wt: &Path) {
2475 Fx::target(wt);
2476 let target = wt.join("target");
2477 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2478 for subdir in ["debug/deps", "debug/.fingerprint"] {
2479 std::fs::create_dir_all(target.join(subdir)).unwrap();
2480 }
2481 std::fs::write(target.join(".rustc_info.json"),
2482 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2483 }
2484
2485 #[tokio::test]
2486 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2487 for legacy in [false, true] {
2488 let f = Fx::new().await;
2489 let mut state = f.run("ac00", true).await;
2490 Fx::target(&state.candidates[0].worktree);
2491 let original = state.updated_at;
2492 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2493 if legacy {
2494 let path = f.home.join("runs").join(&state.id).join("run.json");
2495 let mut value = serde_json::to_value(&state).unwrap();
2496 value.as_object_mut().unwrap().remove("output_activity_at");
2497 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2498 state = RunState::load_under(&state.id, &f.home).unwrap();
2499 assert_eq!(state.output_activity(), original);
2500 }
2501 let age = state.output_activity();
2502 for offset in [30, 20, 10] {
2503 state
2504 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2505 .unwrap();
2506 state = RunState::load_under(&state.id, &f.home).unwrap();
2507 assert_eq!(state.output_activity(), age);
2508 }
2509 let mut disk = Fx::disk();
2510 disk.fold_grace_secs = 3600;
2511 let report = sweep(&f.home, &disk, now).await;
2512 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2513 }
2514 }
2515
2516 #[tokio::test]
2517 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2518 for evidence in [
2519 "execution",
2520 "artifact",
2521 "dep-info",
2522 "fingerprint",
2523 "lease",
2524 "future",
2525 ] {
2526 let f = Fx::new().await;
2527 let mut state = f.run("ac01", true).await;
2528 let wt = state.candidates[0].worktree.clone();
2529 missing_tag_target(&wt);
2530 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2531 let recent = now - jiff::SignedDuration::from_secs(10);
2532 match evidence {
2533 "execution" => {
2534 state.note_output_activity(recent);
2535 state.save_under_at(&f.home, recent).unwrap();
2536 }
2537 "lease" => {
2538 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2539 let file = lease._file.try_clone().unwrap();
2540 drop(lease);
2541 file.set_modified(std::time::SystemTime::from(recent))
2542 .unwrap();
2543 }
2544 _ => {
2545 let relative = match evidence {
2546 "dep-info" => "debug/output.d",
2547 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2548 _ => "debug/output",
2549 };
2550 let path = wt.join("target").join(relative);
2551 let file = OpenOptions::new()
2552 .write(true)
2553 .create(true)
2554 .truncate(false)
2555 .open(path)
2556 .unwrap();
2557 let at = if evidence == "future" {
2558 now + jiff::SignedDuration::from_secs(10)
2559 } else {
2560 recent
2561 };
2562 file.set_modified(std::time::SystemTime::from(at)).unwrap();
2563 }
2564 }
2565 let mut disk = Fx::disk();
2566 disk.fold_grace_secs = 3600;
2567 let report = sweep(&f.home, &disk, now).await;
2568 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2569 assert!(
2570 report.retained.iter().any(|s| s.contains("grace period")
2571 || s.contains("execution activity after inventory")),
2572 "{evidence}: {report:?}"
2573 );
2574 let aged = now + jiff::SignedDuration::from_secs(7200);
2575 let report = sweep(&f.home, &disk, aged).await;
2576 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2577 }
2578 }
2579
2580 #[tokio::test]
2581 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2582 let f = Fx::new().await;
2583 let state = f.run("ac02", true).await;
2584 let wt = &state.candidates[0].worktree;
2585 missing_tag_target(wt);
2586 let target = wt.join("target");
2587 let unknown = target.join("debug/notes");
2588 std::fs::write(&unknown, "operator data").unwrap();
2589 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2590 let path = target.join("debug").join(lock);
2591 let file = OpenOptions::new()
2592 .create(true)
2593 .truncate(false)
2594 .read(true)
2595 .write(true)
2596 .open(&path)
2597 .unwrap();
2598 crate::pr_head::try_lock_file(&file).unwrap();
2599 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2600 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2601 assert!(
2602 report
2603 .retained
2604 .iter()
2605 .any(|s| s.contains(path.to_str().unwrap())),
2606 "{report:?}"
2607 );
2608 }
2609 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2610 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2611 drop(owner);
2612 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2613 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2614 drop(driver);
2615 let output = OutputLease::borrow(wt).unwrap().unwrap();
2616 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2617 drop(output);
2618 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2619 assert_eq!(report.freed_bytes, 200, "{report:?}");
2620 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2621 assert!(!target.join("CACHEDIR.TAG").exists());
2622 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2623 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2624 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2625 }
2626
2627 #[tokio::test]
2628 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2629 let f = Fx::new().await;
2630 let state = f.run("ac04", true).await;
2631 let wt = &state.candidates[0].worktree;
2632 missing_tag_target(wt);
2633 let target = wt.join("target");
2634 let first = target.join("debug/output");
2635 let second = target.join("debug/second");
2636 std::fs::copy(&first, &second).unwrap();
2637 std::fs::write(
2638 second.with_extension("d"),
2639 format!("{}: source\n", second.display()),
2640 )
2641 .unwrap();
2642 let outputs = declared_outputs(&target).unwrap();
2643 assert_eq!(outputs.len(), 2);
2644 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2646 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2647 std::fs::remove_file(first).unwrap();
2648 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2649 assert_eq!(report.freed_bytes, 200, "{report:?}");
2650 assert!(!second.exists());
2651 assert!(second.with_extension("d").exists());
2652 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2653 }
2654
2655 #[tokio::test]
2656 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2657 for invalid in [
2658 "identity",
2659 "deps",
2660 "fingerprint",
2661 "lock",
2662 "dep-info",
2663 "tag-symlink",
2664 "bad-tag",
2665 ] {
2666 let f = Fx::new().await;
2667 let state = f.run("ac03", true).await;
2668 let wt = &state.candidates[0].worktree;
2669 missing_tag_target(wt);
2670 let target = wt.join("target");
2671 match invalid {
2672 "identity" => std::fs::write(
2673 target.join(".rustc_info.json"),
2674 r#"{"rustc_fingerprint":123}"#,
2675 )
2676 .unwrap(),
2677 "deps" | "fingerprint" => {
2678 let path = target.join("debug").join(if invalid == "deps" {
2679 "deps"
2680 } else {
2681 ".fingerprint"
2682 });
2683 std::fs::remove_dir(&path).unwrap();
2684 std::os::unix::fs::symlink(&f.repo, path).unwrap();
2685 }
2686 "lock" => {
2687 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2688 }
2689 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2690 "tag-symlink" => {
2691 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2692 .unwrap()
2693 }
2694 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2695 _ => unreachable!(),
2696 }
2697 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2698 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2699 assert!(target.join("debug/output").is_file());
2700 }
2701 }
2702
2703 #[tokio::test]
2704 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2705 let f = Fx::new().await;
2706 let first = f.run("ba00", true).await;
2707 let second = f.run("ba01", true).await;
2708 for state in [&first, &second] {
2709 Fx::target(&state.candidates[0].worktree);
2710 }
2711 let mut disk = Fx::disk();
2712 disk.worktree_reclaim_bytes = 100;
2713 let out = sweep(&f.home, &disk, Fx::now()).await;
2714 assert_eq!(out.freed_bytes, 200, "{out:?}");
2715 assert!(
2716 out.retained
2717 .contains(&format!("{}: per-pass byte budget spent", second.id)),
2718 "{out:?}"
2719 );
2720 assert!(
2721 !first.candidates[0]
2722 .worktree
2723 .join("target/debug/output")
2724 .exists()
2725 );
2726 assert!(
2727 second.candidates[0]
2728 .worktree
2729 .join("target/debug/output")
2730 .exists()
2731 );
2732 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2733 }
2734
2735 #[tokio::test]
2736 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2737 let f = Fx::new().await;
2738 let small = f.run("bb00", true).await;
2739 let large = f.run("bb01", true).await;
2740 for state in [&small, &large] {
2741 Fx::target(&state.candidates[0].worktree);
2742 }
2743 let output = large.candidates[0].worktree.join("target/debug/output");
2744 std::fs::OpenOptions::new()
2745 .write(true)
2746 .open(&output)
2747 .unwrap()
2748 .set_len(1000)
2749 .unwrap();
2750 let mut disk = Fx::disk();
2751 disk.worktree_reclaim_bytes = 300;
2752 disk.min_free_bytes = u64::MAX;
2753 disk.worktree_limit_bytes = u64::MAX;
2755 let out = sweep(&f.home, &disk, Fx::now()).await;
2756 assert_eq!(out.freed_bytes, 1000, "{out:?}");
2757 assert!(!output.exists());
2758 assert!(
2759 small.candidates[0]
2760 .worktree
2761 .join("target/debug/output")
2762 .exists()
2763 );
2764 }
2765
2766 #[tokio::test]
2767 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2768 let f = Fx::new().await;
2769 let first = f.run("bc00", true).await;
2770 let second = f.run("bc01", true).await;
2771 for state in [&first, &second] {
2772 Fx::target(&state.candidates[0].worktree);
2773 }
2774 std::fs::create_dir_all(
2776 first.candidates[0]
2777 .worktree
2778 .join("target/debug/incremental"),
2779 )
2780 .unwrap();
2781 std::fs::write(
2782 first.candidates[0]
2783 .worktree
2784 .join("target/debug/incremental/cache"),
2785 [0; 1000],
2786 )
2787 .unwrap();
2788 let mut disk = Fx::disk();
2789 disk.worktree_reclaim_bytes = 0;
2790 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2791 disk.worktree_reclaim_bytes = 300;
2792 let out = sweep(&f.home, &disk, Fx::now()).await;
2793 assert_eq!(out.freed_bytes, 200, "{out:?}");
2794 assert!(
2795 !first.candidates[0]
2796 .worktree
2797 .join("target/debug/output")
2798 .exists()
2799 );
2800 assert!(
2801 second.candidates[0]
2802 .worktree
2803 .join("target/debug/output")
2804 .exists()
2805 );
2806 assert!(
2807 out.retained
2808 .iter()
2809 .any(|r| r.contains("declared outputs exceed"))
2810 );
2811 }
2812
2813 #[tokio::test]
2814 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2815 let f = Fx::new().await;
2816 let mut protected = f.run("bd00", true).await;
2817 let later = f.run("bd01", true).await;
2818 for state in [&protected, &later] {
2819 Fx::target(&state.candidates[0].worktree);
2820 }
2821 let mut disk = Fx::disk();
2822 disk.worktree_reclaim_bytes = 100;
2823 disk.fold_grace_secs = 3600;
2824 let out = sweep(&f.home, &disk, Fx::now()).await;
2825 assert_eq!(out.freed_bytes, 0);
2826 assert!(
2827 out.retained
2828 .iter()
2829 .any(|r| r.contains("within grace period"))
2830 );
2831 disk.fold_grace_secs = 0;
2832 protected.driver_exited = false;
2833 protected.driver_pid = Some(std::process::id());
2834 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2835 protected
2836 .save_under_at(&f.home, protected.updated_at)
2837 .unwrap();
2838 let out = sweep(&f.home, &disk, Fx::now()).await;
2839 assert!(
2840 out.retained
2841 .iter()
2842 .any(|r| r.contains("run live or ownership unknown"))
2843 );
2844 assert_eq!(
2845 out.freed_bytes, 200,
2846 "live target must not consume exception: {out:?}"
2847 );
2848 let output = protected.candidates[0].worktree.join("target/debug/output");
2849 assert!(output.exists());
2850 protected.driver_exited = true;
2851 protected.save_under(&f.home).unwrap();
2852 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2853 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2854 assert!(output.exists());
2855 drop(owner);
2856 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2857 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2858 assert!(output.exists());
2859 drop(cargo);
2860 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2861 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2862 assert!(output.exists());
2863 drop(lease);
2864 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2865 assert!(!output.exists());
2866 }
2867
2868 #[tokio::test]
2869 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2870 for remove_root in [true, false] {
2871 let f = Fx::new().await;
2872 let stale = Fx::new().await;
2873 let mut missing = stale.run("aa00", false).await;
2874 Fx::target(&missing.candidates[0].worktree);
2875 missing.save_under(&f.home).unwrap();
2876 if remove_root {
2877 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2878 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2881 } else {
2882 std::fs::remove_dir_all(&stale.repo).unwrap();
2883 }
2884 let healthy = f.run("aa01", false).await;
2885 Fx::target(&healthy.candidates[0].worktree);
2886
2887 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2888 assert!(report.failures.is_empty(), "{report:?}");
2889 assert_eq!(report.freed_bytes, 200, "{report:?}");
2890 assert!(
2891 !healthy.candidates[0]
2892 .worktree
2893 .join("target/debug/output")
2894 .exists()
2895 );
2896 if !remove_root {
2897 assert!(
2899 missing.candidates[0]
2900 .worktree
2901 .join("target/debug/output")
2902 .exists()
2903 );
2904 }
2905 }
2906 }
2907
2908 #[tokio::test]
2909 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2910 let f = Fx::new().await;
2911 let mut missing = f.run("aa00", false).await;
2912 Fx::target(&missing.candidates[0].worktree);
2913 missing.repo = missing.candidates[0].worktree.clone();
2914 missing.save_under(&f.home).unwrap();
2915 std::fs::remove_dir_all(&missing.repo).unwrap();
2916 assert!(missing.worktree_root().is_dir());
2917 assert!(f.repo.is_dir());
2918
2919 let healthy = f.run("aa01", false).await;
2920 Fx::target(&healthy.candidates[0].worktree);
2921
2922 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2923 assert!(report.failures.is_empty(), "{report:?}");
2924 assert_eq!(report.freed_bytes, 200, "{report:?}");
2925 assert!(
2926 !healthy.candidates[0]
2927 .worktree
2928 .join("target/debug/output")
2929 .exists()
2930 );
2931 }
2932
2933 #[tokio::test]
2934 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2935 let f = Fx::new().await;
2936 let broken = Fx::new().await;
2937 let mut invalid = broken.run("aa00", false).await;
2938 Fx::target(&invalid.candidates[0].worktree);
2939 invalid.save_under(&f.home).unwrap();
2940 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2941 let healthy = f.run("aa01", false).await;
2942 Fx::target(&healthy.candidates[0].worktree);
2943
2944 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2945 assert_eq!(report.failures.len(), 1, "{report:?}");
2946 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2947 assert_eq!(report.freed_bytes, 200, "{report:?}");
2948 assert!(
2949 !healthy.candidates[0]
2950 .worktree
2951 .join("target/debug/output")
2952 .exists()
2953 );
2954 assert!(
2955 invalid.candidates[0]
2956 .worktree
2957 .join("target/debug/output")
2958 .exists()
2959 );
2960 }
2961
2962 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2963 let path = reconcile::run(&f.home, &task.id, &source.id, None)
2964 .await
2965 .unwrap();
2966 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2967 }
2968
2969 #[tokio::test]
2970 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
2971 use crate::{
2972 chat_control::{Operation, dispatch},
2973 talk::{self, Talks},
2974 };
2975 let f = Fx::new().await;
2976 let source = f.run("ae00", true).await;
2977 let clean = f.run("ae01", false).await;
2978 let queue = crate::queue::Queue::at(f.home.join("queue"));
2979 let talks = Talks::at(f.home.join("talks"));
2980 let open_talk = |id: &str| -> talk::Talk {
2981 let mut conversation = serde_json::from_value(serde_json::json!({
2982 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
2983 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
2984 "seat":crate::agent::SeatState::new("talk","worker",1)
2985 }))
2986 .unwrap();
2987 talks.put(&mut conversation).unwrap();
2988 conversation
2989 };
2990 let conversation = open_talk("talk-owner");
2991 let other = open_talk("talk-other");
2992 let task = f.task(&[source.clone(), clean.clone()]);
2994 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2995 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
2996 task: task.id.clone(),
2997 authority: source.id.clone(),
2998 apply: apply.map(str::to_owned),
2999 };
3000 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
3002 .await
3003 .unwrap();
3004 let legacy: serde_json::Value =
3005 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
3006 let legacy_id = legacy["id"].as_str().unwrap();
3007 assert!(
3008 dispatch(
3009 &queue,
3010 &f.home,
3011 &talks,
3012 &conversation.id,
3013 operation(Some(legacy_id))
3014 )
3015 .await
3016 .is_err()
3017 );
3018 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
3019 .await
3020 .unwrap();
3021 assert!(summary.contains("Complete review summary"));
3022 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
3023 let root = f.home.join("retention-reconciliations");
3024 let displayed: serde_json::Value =
3025 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
3026 let id = displayed["plan"].as_str().unwrap();
3027 let dir = root.join(id);
3028 let plan: serde_json::Value =
3029 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
3030 assert_eq!(plan["version"], 2);
3031 assert!(clean.candidates[0].worktree.exists());
3032 assert!(!dir.join("approved.json").exists());
3033 assert!(
3035 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
3036 .await
3037 .is_err()
3038 );
3039 assert!(!dir.join("approved.json").exists());
3040 let error = reconcile::interrupt_chat_after_fence(
3043 &f.home,
3044 &task,
3045 id,
3046 reconcile::ChatRequest {
3047 talks: &talks,
3048 conversation: &conversation.id,
3049 },
3050 )
3051 .await
3052 .unwrap_err();
3053 assert!(
3054 error.to_string().contains("simulated interruption"),
3055 "{error:#}"
3056 );
3057 assert!(clean.candidates[0].worktree.exists());
3058 assert!(dir.join("approved.json").exists());
3059 dispatch(
3060 &queue,
3061 &f.home,
3062 &talks,
3063 &conversation.id,
3064 operation(Some(id)),
3065 )
3066 .await
3067 .unwrap();
3068 assert!(!clean.candidates[0].worktree.exists());
3069 assert!(source.candidates[0].worktree.exists());
3070 let audit: serde_json::Value =
3071 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3072 assert_eq!(audit["version"], 2);
3073 assert_eq!(
3074 audit["owner"],
3075 serde_json::json!({"conversation": conversation.id})
3076 );
3077 dispatch(
3078 &queue,
3079 &f.home,
3080 &talks,
3081 &conversation.id,
3082 operation(Some(id)),
3083 )
3084 .await
3085 .unwrap();
3086 }
3087
3088 #[tokio::test]
3089 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3090 let f = Fx::new().await;
3091 let source = f.run("af00", true).await;
3092 let mut clean = f.run("af01", false).await;
3093 let mut dirty = f.run("af02", false).await;
3094 for state in [&mut clean, &mut dirty] {
3095 state.status = RunStatus::Prep;
3096 state.prep_complete = Some(false);
3097 state.candidates[0].empty = false;
3098 state.save_under(&f.home).unwrap();
3099 }
3100 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3101 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3102 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3103 let plan = reconciliation_plan(&f, &task, &source).await;
3104 let selected: Vec<_> = plan["candidates"]
3105 .as_array()
3106 .unwrap()
3107 .iter()
3108 .filter(|e| e["selected"] == true)
3109 .collect();
3110 assert_eq!(selected.len(), 1, "{plan:#}");
3111 assert_eq!(selected[0]["run"], clean.id);
3112 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3113 .await
3114 .unwrap();
3115 assert!(!clean.candidates[0].worktree.exists());
3116 assert!(dirty.candidates[0].worktree.exists());
3117 assert!(
3118 RunState::load_under(&dirty.id, &f.home)
3119 .unwrap()
3120 .retired_to
3121 .is_none()
3122 );
3123 }
3124
3125 #[tokio::test]
3126 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3127 let f = Fx::new().await;
3128 let mut source = f.run("aa00", true).await;
3129 let alternative = f.run("aa01", true).await;
3130 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3131 .await
3132 .unwrap();
3133 source.reviewed_commits = Some(vec![head.clone()]);
3134 source.save_under(&f.home).unwrap();
3135 let mut runs = vec![source.clone(), alternative.clone()];
3136 for i in 2..14 {
3137 runs.push(f.run(&format!("aa{i:02}"), false).await);
3138 }
3139 let mut task = f.task(&runs);
3140 task.pr_recovery = Some(crate::pr_recovery::Selection {
3141 url: "https://example.invalid/pull/1".into(),
3142 branch: "selected".into(),
3143 head: head.clone(),
3144 source_head: Some(head.clone()),
3145 source_run: Some(source.id.clone()),
3146 source_proof: None,
3147 disposition: Some(crate::pr_recovery::Disposition::Land),
3148 instruction: task.instruction.clone(),
3149 approved_at: Timestamp::now(),
3150 });
3151 source.pr_recovery = task.pr_recovery.clone();
3152 source.save_under(&f.home).unwrap();
3153 runs[0] = source.clone();
3154 crate::queue::Queue::at(f.home.join("queue"))
3155 .put(&mut task)
3156 .unwrap();
3157 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3158 let task_before = serde_json::to_value(&task).unwrap();
3159 let mut report = Report::default();
3160 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3161 assert_eq!(
3162 report.retired, 0,
3163 "automatic cleanup must still refuse blanket markers"
3164 );
3165 assert!(
3166 report
3167 .retained
3168 .iter()
3169 .any(|reason| reason.contains("explicitly retained"))
3170 );
3171 let mut total = 0;
3172 for expected in [5, 5, 2, 0] {
3173 let plan = reconciliation_plan(&f, &task, &source).await;
3174 let selected: Vec<_> = plan["candidates"]
3175 .as_array()
3176 .unwrap()
3177 .iter()
3178 .filter(|e| e["selected"] == true)
3179 .collect();
3180 assert_eq!(selected.len(), expected, "{plan:#}");
3181 for e in &selected {
3182 assert!(
3183 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3184 .unwrap()
3185 .retired_to
3186 .is_none()
3187 );
3188 }
3189 let operation = plan["id"].as_str().unwrap();
3190 for _ in 0..2 {
3191 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3192 .await
3193 .unwrap();
3194 }
3195 total += expected;
3196 }
3197 assert_eq!(total, 12);
3198 for run in &runs {
3199 let dir = f.home.join("runs").join(&run.id);
3200 assert!(dir.join("run.json").exists());
3201 assert!(dir.join("retain-recovery-sources").exists());
3202 assert!(
3203 git::branch_exists(&f.repo, &run.candidates[0].branch)
3204 .await
3205 .unwrap()
3206 );
3207 }
3208 assert!(source.candidates[0].worktree.exists());
3209 assert!(alternative.candidates[0].worktree.exists());
3210 assert_eq!(
3211 git::rev_parse(&f.repo, &source.candidates[0].branch)
3212 .await
3213 .unwrap(),
3214 head
3215 );
3216 assert_eq!(
3217 serde_json::to_value(
3218 crate::queue::Queue::at(f.home.join("queue"))
3219 .get(&task.id)
3220 .unwrap()
3221 )
3222 .unwrap(),
3223 task_before
3224 );
3225 assert_eq!(
3226 runs.iter()
3227 .filter(|r| RunState::load_under(&r.id, &f.home)
3228 .unwrap()
3229 .retired_to
3230 .is_none())
3231 .count(),
3232 2
3233 );
3234 }
3235
3236 #[tokio::test]
3237 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3238 let f = Fx::new().await;
3239 let source = f.run("ab00", true).await;
3240 let mut runs = vec![source.clone()];
3241 for i in 1..9 {
3242 runs.push(f.run(&format!("ab{i:02}"), false).await);
3243 }
3244 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3245 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3246 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3247 std::fs::write(
3248 runs[3].candidates[0].worktree.join("scratch/source"),
3249 "ignored source",
3250 )
3251 .unwrap();
3252 std::fs::write(
3253 runs[4].worktree_root().join("unknown"),
3254 "unregistered source",
3255 )
3256 .unwrap();
3257 let checkpoint = git::git(
3258 &runs[5].candidates[0].worktree,
3259 &[
3260 "rev-parse",
3261 "--path-format=absolute",
3262 "--git-path",
3263 "sequencer",
3264 ],
3265 )
3266 .await
3267 .unwrap();
3268 std::fs::create_dir(checkpoint.trim()).unwrap();
3269 runs[6].driver_exited = false;
3270 runs[6].driver_pid = Some(std::process::id());
3271 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3272 runs[6].save_under(&f.home).unwrap();
3273 runs[7].driver_exited = false;
3274 runs[7].driver_pid = None;
3275 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3277 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3278
3279 let hidden = &runs[8].candidates[0].worktree;
3280 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3281 .await
3282 .unwrap();
3283 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3284 assert!(
3285 git::git(hidden, &["status", "--porcelain"])
3286 .await
3287 .unwrap()
3288 .is_empty()
3289 );
3290 let task = f.task(&runs);
3291 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3292 let plan = reconciliation_plan(&f, &task, &source).await;
3293 assert!(
3294 plan["candidates"]
3295 .as_array()
3296 .unwrap()
3297 .iter()
3298 .all(|e| e["selected"] == false),
3299 "{plan:#}"
3300 );
3301 for run in &runs {
3302 assert!(run.candidates[0].worktree.exists());
3303 }
3304 }
3305
3306 #[tokio::test]
3307 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3308 let f = Fx::new().await;
3309 let source = f.run("ac00", true).await;
3310 let empty = f.run("ac01", false).await;
3311 let task = f.task(&[source.clone(), empty.clone()]);
3312 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3313 let plan = reconciliation_plan(&f, &task, &source).await;
3314 std::fs::write(
3315 f.home.join("runs").join(&source.id).join("new-report"),
3316 "new review checkpoint",
3317 )
3318 .unwrap();
3319 assert!(
3320 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3321 .await
3322 .is_err()
3323 );
3324 assert!(empty.candidates[0].worktree.exists());
3325 assert!(
3326 RunState::load_under(&empty.id, &f.home)
3327 .unwrap()
3328 .retired_to
3329 .is_none()
3330 );
3331 let marker = f
3332 .home
3333 .join("runs")
3334 .join(&empty.id)
3335 .join("retain-recovery-sources");
3336 std::fs::write(&marker, "different owner").unwrap();
3337 let plan = reconciliation_plan(&f, &task, &source).await;
3338 assert!(
3339 plan["candidates"]
3340 .as_array()
3341 .unwrap()
3342 .iter()
3343 .all(|e| e["selected"] == false)
3344 );
3345 }
3346
3347 #[tokio::test]
3348 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3349 for partial in [false, true] {
3350 let f = Fx::new().await;
3351 let source = f.run("ab10", true).await;
3352 let first = f.run("ab11", false).await;
3353 let second = f.run("ab12", false).await;
3354 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3355 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3356 let plan = reconciliation_plan(&f, &task, &source).await;
3357 let id = plan["id"].as_str().unwrap();
3358 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3359 std::fs::write(&staging, b"").unwrap();
3361 assert!(
3362 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3363 .await
3364 .is_err()
3365 );
3366 assert!(staging.exists());
3367 std::fs::remove_file(&staging).unwrap();
3368 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3369 .await
3370 .unwrap_err();
3371 assert!(
3372 error.to_string().contains("before run state rename"),
3373 "{error:#}"
3374 );
3375 assert!(
3376 RunState::load_under(&first.id, &f.home)
3377 .unwrap()
3378 .retired_to
3379 .is_some()
3380 );
3381 assert!(
3382 RunState::load_under(&second.id, &f.home)
3383 .unwrap()
3384 .retired_to
3385 .is_none()
3386 );
3387 assert!(first.candidates[0].worktree.exists());
3388 assert!(second.candidates[0].worktree.exists());
3389 let interrupted = std::fs::read(&staging).unwrap();
3390 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3392 assert!(
3393 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3394 .await
3395 .is_err()
3396 );
3397 assert_eq!(
3398 std::fs::read(&staging).unwrap(),
3399 b"unrelated recovery source"
3400 );
3401 std::fs::write(&staging, interrupted).unwrap();
3402 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3403 .await
3404 .unwrap();
3405 assert!(!staging.exists());
3406 assert!(!first.candidates[0].worktree.exists());
3407 assert!(!second.candidates[0].worktree.exists());
3408 assert_eq!(
3409 RunState::load_under(&second.id, &f.home)
3410 .unwrap()
3411 .retired_to,
3412 Some(source.id)
3413 );
3414 }
3415 }
3416
3417 #[tokio::test]
3418 async fn reconciliation_resumes_only_its_exact_durable_fence() {
3419 let f = Fx::new().await;
3420 let source = f.run("ad00", true).await;
3421 let empty = f.run("ad01", false).await;
3422 let task = f.task(&[source.clone(), empty.clone()]);
3423 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3424 let plan = reconciliation_plan(&f, &task, &source).await;
3425 let id = plan["id"].as_str().unwrap();
3426 assert!(
3427 reconcile::interrupt_after_fence(&f.home, &task, id)
3428 .await
3429 .is_err()
3430 );
3431 assert!(empty.candidates[0].worktree.exists());
3432 assert_eq!(
3433 RunState::load_under(&empty.id, &f.home)
3434 .unwrap()
3435 .retired_to
3436 .as_ref(),
3437 Some(&source.id)
3438 );
3439 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3440 .await
3441 .unwrap();
3442 assert!(!empty.candidates[0].worktree.exists());
3443 git::worktree_add_branch(
3445 &f.repo,
3446 &empty.candidates[0].worktree,
3447 "restore-empty",
3448 &f.base,
3449 )
3450 .await
3451 .unwrap();
3452 assert!(
3454 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3455 .await
3456 .is_err()
3457 );
3458 assert!(empty.candidates[0].worktree.exists());
3459 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3460 changed.prep_failure = Some("new owner edit".into());
3461 changed.save_under(&f.home).unwrap();
3462 assert!(
3463 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3464 .await
3465 .is_err()
3466 );
3467 }
3468
3469 #[tokio::test]
3470 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3471 let f = Fx::new().await;
3472 let source = f.run("ae00", true).await;
3473 let mut prep = f.run("ae01", false).await;
3474 let candidate = prep.candidates.pop().unwrap();
3475 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3476 prep.save_under(&f.home).unwrap();
3477 let reset = f.run("ae02", false).await;
3478 let wt = &reset.candidates[0].worktree;
3479 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3480 git::git(wt, &["add", "."]).await.unwrap();
3481 git::git(wt, &["commit", "-m", "unique implementation"])
3482 .await
3483 .unwrap();
3484 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3485 let unknown = f.run("ae03", false).await;
3486 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3487 let mut raw: serde_json::Value =
3488 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3489 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3490 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3491 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3492 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3493 let plan = reconciliation_plan(&f, &task, &source).await;
3494 let selected: Vec<_> = plan["candidates"]
3495 .as_array()
3496 .unwrap()
3497 .iter()
3498 .filter(|e| e["selected"] == true)
3499 .collect();
3500 assert_eq!(selected.len(), 1, "{plan:#}");
3501 assert_eq!(selected[0]["run"], prep.id);
3502 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3503 .await
3504 .unwrap();
3505 assert!(!candidate.worktree.exists());
3506 assert!(reset.candidates[0].worktree.exists());
3507 assert!(unknown.candidates[0].worktree.exists());
3508 }
3509
3510 #[tokio::test]
3511 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3512 let f = Fx::new().await;
3513 let mut state = f.run("aaaa", true).await;
3514 let candidate = state.candidates[0].worktree.clone();
3515 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3516 Fx::target(&candidate);
3517 for seat in ["review-1", "review-2"] {
3518 let wt = state.worktree_root().join(seat);
3519 git::worktree_add_detached(&f.repo, &wt, &f.base)
3520 .await
3521 .unwrap();
3522 Fx::target(&wt);
3523 }
3524 state.status = RunStatus::Landing;
3525 state.save_under(&f.home).unwrap();
3526 let second = Fx::new().await;
3527 let other = second.run("bbbb", true).await;
3528 let mut imported = other.clone();
3530 imported.save_under(&f.home).unwrap();
3531 Fx::target(&other.candidates[0].worktree);
3532 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3533 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3534 assert!(
3535 candidate.join("target/debug/output").exists(),
3536 "active writer protected"
3537 );
3538 assert!(
3539 !state
3540 .worktree_root()
3541 .join("review-1/target/debug/output")
3542 .exists()
3543 );
3544 assert!(
3545 !state
3546 .worktree_root()
3547 .join("review-2/target/debug/output")
3548 .exists()
3549 );
3550 assert!(
3551 !other.candidates[0]
3552 .worktree
3553 .join("target/debug/output")
3554 .exists()
3555 );
3556 assert!(out.freed_bytes >= 600);
3557 drop(guard);
3558 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3559 assert!(out.freed_bytes >= 200);
3560 assert_eq!(
3561 std::fs::read_to_string(candidate.join("source")).unwrap(),
3562 "uncommitted work"
3563 );
3564 assert!(
3565 git::branch_exists(&f.repo, &state.candidates[0].branch)
3566 .await
3567 .unwrap()
3568 );
3569 }
3570
3571 #[tokio::test]
3572 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3573 let f = Fx::new().await;
3574 let state = f.run("cc01", false).await;
3575 let wt = &state.candidates[0].worktree;
3576 Fx::target(wt);
3577 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3578 let dir = wt.join("target").join(dir);
3579 std::fs::create_dir_all(&dir).unwrap();
3580 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3581 }
3582 let manifest = wt.join("target/debug/output.d");
3584 let output = wt.join("target/debug/output");
3585 std::fs::write(
3586 &manifest,
3587 format!(
3588 "{}: {}\n",
3589 output.display(),
3590 wt.join("target/debug/notes.txt").display()
3591 ),
3592 )
3593 .unwrap();
3594 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3595 assert_eq!(report.freed_bytes, 200);
3596 assert!(!output.exists());
3597 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3598 assert_eq!(
3599 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3600 "user evidence"
3601 );
3602 }
3603 assert!(manifest.exists());
3605 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3606 std::fs::write(&output, "user note replacing an old binary").unwrap();
3607 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3608 assert_eq!(report.freed_bytes, 0);
3609 assert_eq!(
3610 std::fs::read_to_string(&output).unwrap(),
3611 "user note replacing an old binary"
3612 );
3613 }
3614
3615 #[tokio::test]
3616 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3617 let f = Fx::new().await;
3618 let state = f.run("cc02", false).await;
3619 let wt = &state.candidates[0].worktree;
3620 Fx::target(wt);
3621 let mut disk = Fx::disk();
3622 disk.worktree_limit_bytes = u64::MAX;
3623 let now = Fx::now();
3624 let first = sweep(&f.home, &disk, now).await;
3625 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3626 assert_eq!(
3627 sweep(&f.home, &disk, now).await.measured_bytes,
3628 first.measured_bytes
3629 );
3630 assert!(
3631 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3632 .await
3633 .measured_bytes
3634 > first.measured_bytes
3635 );
3636 disk.worktree_limit_bytes = 0;
3637 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3638 }
3639
3640 #[tokio::test]
3641 async fn unfinished_source_fold_reports_its_retention_reason() {
3642 let f = Fx::new().await;
3643 let mut source = f.run("cc03", true).await;
3644 let task = f.task(&[source.clone()]);
3645 let error = crate::graph::fold_run(&mut source, true, &f.home)
3646 .await
3647 .unwrap_err();
3648 assert!(error.to_string().contains(&task.id));
3649 assert!(source.candidates[0].worktree.exists());
3650 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3651 let error = crate::graph::fold_run(&mut source, true, &f.home)
3652 .await
3653 .unwrap_err();
3654 assert!(error.to_string().contains("explicitly retained"));
3655 }
3656
3657 #[tokio::test]
3658 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3659 let f = Fx::new().await;
3660 let mut source = f.run("cc04", true).await;
3661 let mut task = f.task(&[source.clone()]);
3662 task.hold_manual(Some("queue settlement pending".into()));
3663 let queue = crate::queue::Queue::at(f.home.join("queue"));
3664 queue.put(&mut task).unwrap();
3665 source.merge = Some(crate::run::MergeOutcome {
3666 mode: crate::config::MergeMode::Pr,
3667 ok: true,
3668 detail: "confirmed merged".into(),
3669 empty: false,
3670 });
3671 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3673 source.status = status;
3674 assert!(
3675 crate::graph::fold_run(&mut source, true, &f.home)
3676 .await
3677 .is_err()
3678 );
3679 assert!(source.candidates[0].worktree.exists());
3680 }
3681 git::git(
3682 &f.repo,
3683 &["merge", "--ff-only", &source.candidates[0].branch],
3684 )
3685 .await
3686 .unwrap();
3687 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3688 .await
3689 .unwrap();
3690 source.status = RunStatus::Merged;
3691 source.save_under(&f.home).unwrap();
3692 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3693 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3694 assert!(
3695 crate::graph::fold_run(&mut source, true, &f.home)
3696 .await
3697 .unwrap_err()
3698 .to_string()
3699 .contains("explicitly retained")
3700 );
3701 std::fs::remove_file(
3704 f.home
3705 .join("runs")
3706 .join(&source.id)
3707 .join("retain-recovery-sources"),
3708 )
3709 .unwrap();
3710 let (folded, unreadable) = crate::clean::fold_due(
3711 &f.home.join("runs"),
3712 &f.home,
3713 &f.bay,
3714 &Fx::disk(),
3715 Fx::now(),
3716 )
3717 .await
3718 .unwrap();
3719 assert_eq!((folded, unreadable), (1, 0));
3720 assert!(!source.candidates[0].worktree.exists());
3721 let evidence = f
3722 .home
3723 .join("runs")
3724 .join(&source.id)
3725 .join("artifacts/fold-evidence");
3726 let names: Vec<_> = std::fs::read_dir(evidence)
3727 .unwrap()
3728 .map(|entry| entry.unwrap().file_name())
3729 .collect();
3730 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3731 assert_eq!(
3732 queue.get(&task.id).unwrap().status,
3733 crate::queue::TaskStatus::Held
3734 );
3735 }
3736
3737 #[tokio::test]
3738 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3739 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3740 let f = Fx::new().await;
3741 let mut state = f.run("cc06", true).await;
3742 let candidate = state.candidates[0].clone();
3743 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3744 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3745 git::rescue_commit(&candidate.worktree, "second candidate change")
3746 .await
3747 .unwrap();
3748 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3749 state.tally = Some(crate::run::Tally {
3750 first_choice: Default::default(),
3751 borda: Default::default(),
3752 winner: 'A',
3753 rankings: 0,
3754 unanimous_initial: true,
3755 deliberated: false,
3756 changed_votes: 0,
3757 unanimous_final: true,
3758 tie_break: None,
3759 judges: 0,
3760 present: 0,
3761 quorum: 0,
3762 met_quorum: true,
3763 uncontested: Some("fixture".into()),
3764 });
3765 let reviewer = state.worktree_root().join("review-1");
3768 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3769 .await
3770 .unwrap();
3771 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3774 git::rescue_commit(&f.repo, "advance before squash")
3775 .await
3776 .unwrap();
3777 let published = if variant == "partial" {
3778 &intermediate
3779 } else {
3780 &head
3781 };
3782 git::git(&f.repo, &["merge", "--squash", published])
3783 .await
3784 .unwrap();
3785 git::git(&f.repo, &["commit", "-m", "squash candidate"])
3786 .await
3787 .unwrap();
3788 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3789 assert!(
3790 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3791 .await
3792 .unwrap()
3793 .ok()
3794 );
3795 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3796 git::rescue_commit(&f.repo, "advance after squash")
3797 .await
3798 .unwrap();
3799 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3800 .await
3801 .unwrap();
3802 if matches!(variant, "checkpoint" | "reviewer") {
3803 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3805 git::rescue_commit(&reviewer, "unpublished checkpoint")
3806 .await
3807 .unwrap();
3808 if variant == "checkpoint" {
3809 git::git(&reviewer, &["reset", "--hard", &intermediate])
3810 .await
3811 .unwrap();
3812 }
3813 }
3814 if variant == "loser" {
3815 let mut loser = candidate.clone();
3816 loser.label = 'B';
3817 loser.branch = "magi/cc06/B".into();
3818 loser.worktree = state.worktree_root().join("cand-B");
3819 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3820 .await
3821 .unwrap();
3822 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3823 git::rescue_commit(&loser.worktree, "independent loser")
3824 .await
3825 .unwrap();
3826 state.candidates.push(loser);
3827 }
3828 state.status = RunStatus::Merged;
3829 state.merge = Some(crate::run::MergeOutcome {
3830 mode: crate::config::MergeMode::Pr,
3831 ok: true,
3832 detail: "candidate A squash merged".into(),
3833 empty: false,
3834 });
3835 state.save_under(&f.home).unwrap();
3836 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3837 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3838 result.unwrap();
3839 assert!(
3840 !candidate.worktree.exists() && !reviewer.exists(),
3841 "{variant}"
3842 );
3843 assert!(
3844 !git::branch_exists(&f.repo, &candidate.branch)
3845 .await
3846 .unwrap()
3847 );
3848 let artifacts = f
3849 .home
3850 .join("runs")
3851 .join(&state.id)
3852 .join("artifacts/fold-evidence");
3853 assert_eq!(
3854 artifacts.join(format!("{head}.patch")).exists(),
3855 variant == "partial",
3856 "only a partially published winner needs evidence: {variant}"
3857 );
3858 assert_eq!(
3859 artifacts.join(format!("{intermediate}.patch")).exists(),
3860 variant == "partial",
3861 "squash proof also covers intermediate reviewer tips: {variant}"
3862 );
3863 let patches: Vec<_> = std::fs::read_dir(&artifacts)
3864 .unwrap()
3865 .map(|entry| entry.unwrap().path())
3866 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3867 .collect();
3868 let expected = match variant {
3869 "partial" => 2,
3870 "loser" | "reviewer" => 1,
3871 _ => 0,
3872 };
3873 assert_eq!(patches.len(), expected, "{variant}");
3874 if variant == "loser" {
3875 assert!(
3876 std::fs::read_to_string(&patches[0])
3877 .unwrap()
3878 .contains("losing candidate")
3879 );
3880 }
3881 assert!(artifacts.join("manifest.json").exists());
3882 if variant == "reviewer" {
3883 assert!(
3884 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3885 .unwrap()
3886 .contains("reviewer checkpoint")
3887 );
3888 }
3889 crate::graph::fold_run(&mut state, true, &f.home)
3890 .await
3891 .unwrap();
3892 }
3893 }
3894
3895 #[tokio::test]
3896 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3897 for missing in [true, false] {
3898 let f = Fx::new().await;
3899 let mut state = f.run("cc05", true).await;
3900 git::git(
3903 &f.repo,
3904 &["update-ref", "refs/remotes/origin/main", &f.base],
3905 )
3906 .await
3907 .unwrap();
3908 let candidate = state.candidates[0].clone();
3909 if !missing {
3910 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3911 git::rescue_commit(&f.repo, "advance base before rebase")
3912 .await
3913 .unwrap();
3914 git::git(&candidate.worktree, &["rebase", "main"])
3915 .await
3916 .unwrap();
3917 git::sync_to_head(&candidate.worktree).await.unwrap();
3920 }
3921 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3922 state.reviewed_commits = Some(Vec::new());
3924 state.candidates[0].worktree = state.worktree_root().join("under-review");
3925 git::git(
3926 &f.repo,
3927 &[
3928 "worktree",
3929 "move",
3930 candidate.worktree.to_str().unwrap(),
3931 state.candidates[0].worktree.to_str().unwrap(),
3932 ],
3933 )
3934 .await
3935 .unwrap();
3936 let reviewer = state.worktree_root().join("review-1");
3937 git::worktree_add_detached(&f.repo, &reviewer, &head)
3938 .await
3939 .unwrap();
3940 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3943 git::rescue_commit(&reviewer, "old review history")
3944 .await
3945 .unwrap();
3946 git::git(&reviewer, &["reset", "--hard", &head])
3947 .await
3948 .unwrap();
3949 if missing {
3950 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3951 }
3952 let run_dir = f.home.join("runs").join(&state.id);
3953 let artifacts = run_dir.join("artifacts");
3954 std::fs::create_dir_all(&artifacts).unwrap();
3955 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3956 if missing {
3957 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3958 }
3959 state.status = RunStatus::Merged;
3960 state.merge = Some(crate::run::MergeOutcome {
3961 mode: crate::config::MergeMode::Pr,
3962 ok: true,
3963 detail: "confirmed squash merge".into(),
3964 empty: false,
3965 });
3966 state.save_under(&f.home).unwrap();
3967 let owned = state.candidates[0].worktree.clone();
3969 state.candidates[0].worktree = f.home.join("missing-external");
3970 assert!(
3971 crate::graph::fold_run(&mut state, true, &f.home)
3972 .await
3973 .unwrap_err()
3974 .to_string()
3975 .contains("outside the inventoried")
3976 );
3977 state.candidates[0].worktree = owned;
3978 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
3980 assert!(
3981 crate::graph::fold_run(&mut state, true, &f.home)
3982 .await
3983 .is_err()
3984 );
3985 assert!(reviewer.exists());
3986 drop(guard);
3987 let evidence = artifacts.join("fold-evidence");
3989 std::fs::write(&evidence, "blocked").unwrap();
3990 assert!(
3991 crate::graph::fold_run(&mut state, true, &f.home)
3992 .await
3993 .is_err()
3994 );
3995 assert!(reviewer.exists());
3996 std::fs::remove_file(&evidence).unwrap();
3997 std::fs::create_dir_all(&evidence).unwrap();
3998 let oversized = evidence.join("capacity-fixture");
3999 std::fs::File::create(&oversized)
4000 .unwrap()
4001 .set_len(64 * 1024 * 1024)
4002 .unwrap();
4003 assert!(
4004 crate::graph::fold_run(&mut state, true, &f.home)
4005 .await
4006 .is_err()
4007 );
4008 assert!(reviewer.exists());
4009 std::fs::remove_file(oversized).unwrap();
4010 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
4013 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
4014 assert!(
4015 crate::graph::fold_run(&mut state, true, &f.home)
4016 .await
4017 .is_err()
4018 );
4019 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
4020 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
4021 .await
4022 .unwrap();
4023 std::fs::write(&checkpoint, &head).unwrap();
4024 assert!(
4025 crate::graph::fold_run(&mut state, true, &f.home)
4026 .await
4027 .is_err()
4028 );
4029 std::fs::remove_file(checkpoint).unwrap();
4030 let (folded, unreadable) = crate::clean::fold_due(
4031 &f.home.join("runs"),
4032 &f.home,
4033 &f.bay,
4034 &Fx::disk(),
4035 Fx::now(),
4036 )
4037 .await
4038 .unwrap();
4039 if folded != 1 {
4040 panic!(
4041 "normal cleanup failed: {:#}",
4042 crate::graph::fold_run(&mut state, true, &f.home)
4043 .await
4044 .unwrap_err()
4045 );
4046 }
4047 assert_eq!((folded, unreadable), (1, 0));
4048 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4049 state = RunState::load_under(&state.id, &f.home).unwrap();
4050 assert!(state.candidates[0].folded);
4051 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4052 crate::graph::fold_run(&mut state, true, &f.home)
4053 .await
4054 .unwrap();
4055 assert_eq!(
4056 std::fs::read(evidence.join("manifest.json")).unwrap(),
4057 manifest
4058 );
4059 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4060 .await
4061 .unwrap();
4062 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4063 assert!(
4064 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4065 .await
4066 .unwrap()
4067 .ok()
4068 );
4069 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4072 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4073 .as_str()
4074 .unwrap();
4075 let recovered = f.home.join("recovered");
4076 std::fs::create_dir_all(&recovered).unwrap();
4077 git::git(&recovered, &["init"]).await.unwrap();
4078 std::fs::write(
4079 recovered.join("source"),
4080 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4081 )
4082 .unwrap();
4083 git::git(
4084 &recovered,
4085 &[
4086 "apply",
4087 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4088 ],
4089 )
4090 .await
4091 .unwrap();
4092 assert_eq!(
4093 std::fs::read_to_string(recovered.join("source")).unwrap(),
4094 "cc05"
4095 );
4096 assert!(
4097 !std::fs::read(evidence.join(format!("{head}.patch")))
4098 .unwrap()
4099 .is_empty()
4100 );
4101 assert_eq!(
4102 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4103 "review findings"
4104 );
4105 if missing {
4106 assert_eq!(
4107 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4108 "original snapshot"
4109 );
4110 } else {
4111 assert!(!artifacts.join("cand-A.patch").exists());
4112 }
4113 assert!(run_dir.join("run.json").exists());
4114 }
4115 }
4116
4117 #[tokio::test]
4118 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4119 use std::os::unix::fs::PermissionsExt as _;
4120 let f = Fx::new().await;
4121 let source = f.run("dd03", true).await;
4122 let empty = f.run("dd04", false).await;
4123 let task = f.task(&[source, empty.clone()]);
4124 let entered = f.repo.join("proof-entered");
4125 let release = f.repo.join("proof-release");
4126 let hook = f.repo.join("proof-monitor");
4127 std::fs::write(&hook, format!(
4128 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4129 entered.display(), release.display())).unwrap();
4130 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4131 git::git(
4132 &f.repo,
4133 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4134 )
4135 .await
4136 .unwrap();
4137 struct ReleaseProof(PathBuf);
4138 impl Drop for ReleaseProof {
4139 fn drop(&mut self) {
4140 let _ = std::fs::write(&self.0, "continue");
4141 }
4142 }
4143 let _release_on_panic = ReleaseProof(release.clone());
4144 let home = f.home.clone();
4145 let sweep = tokio::spawn(async move {
4146 let mut report = Report::default();
4147 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4148 report
4149 });
4150 let watchdog = std::time::Duration::from_secs(60);
4153 tokio::time::timeout(watchdog, async {
4154 while !entered.exists() {
4155 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4156 }
4157 })
4158 .await
4159 .expect("Git retirement proof did not enter the blocking hook");
4160 let queue = crate::queue::Queue::at(f.home.join("queue"));
4161 let mut selected = queue.get(&task.id).unwrap();
4162 selected.resume_override = Some(crate::queue::OperatorResume {
4163 question_id: "during-slow-proof".into(),
4164 at: Fx::now(),
4165 conductor_rehold: None,
4166 forced: false,
4167 pinned_run: Some(empty.id.clone()),
4168 });
4169 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4170 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4174 Ok(result) => {
4175 result.unwrap().unwrap();
4176 true
4177 }
4178 Err(_) => {
4179 std::fs::write(&release, "continue").unwrap();
4180 update.await.unwrap().unwrap();
4181 false
4182 }
4183 };
4184 std::fs::write(release, "continue").unwrap();
4185 let report = tokio::time::timeout(watchdog, sweep)
4186 .await
4187 .expect("Git retirement proof did not finish after release")
4188 .unwrap();
4189 assert!(
4190 updated_while_blocked,
4191 "task writers must not wait behind Git retirement proof"
4192 );
4193 assert_eq!(report.retired, 0);
4194 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4195 assert!(empty.candidates[0].worktree.exists());
4196 assert!(
4197 RunState::load_under(&empty.id, &f.home)
4198 .unwrap()
4199 .retired_to
4200 .is_none()
4201 );
4202 }
4203
4204 #[tokio::test]
4205 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4206 let f = Fx::new().await;
4207 let source = f.run("dd01", true).await;
4208 let mut empty = f.run("dd02", false).await;
4209 let expected = f.task(&[source.clone(), empty.clone()]);
4210 let queue = crate::queue::Queue::at(f.home.join("queue"));
4211 let _claim = queue.claim(&expected.id).unwrap();
4212 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4214 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4215 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4216 let mut selected = queue.get(&expected.id).unwrap();
4217 selected.resume_override = Some(crate::queue::OperatorResume {
4218 question_id: "new-owner-decision".into(),
4219 at: Fx::now(),
4220 conductor_rehold: None,
4221 forced: false,
4222 pinned_run: Some(empty.id.clone()),
4223 });
4224 queue.put(&mut selected).unwrap();
4225 let error = commit_retirement(
4226 &f.home,
4227 &queue,
4228 &expected,
4229 &mut empty,
4230 &source.id,
4231 &expected.runs,
4232 Fx::now(),
4233 )
4234 .unwrap_err();
4235 assert!(error.to_string().contains("task changed"));
4236 assert!(
4237 RunState::load_under(&empty.id, &f.home)
4238 .unwrap()
4239 .retired_to
4240 .is_none()
4241 );
4242 assert!(empty.candidates[0].worktree.exists());
4243 assert_eq!(
4244 queue
4245 .get(&expected.id)
4246 .unwrap()
4247 .resume_override
4248 .unwrap()
4249 .pinned_run,
4250 Some(empty.id.clone())
4251 );
4252 assert!(
4253 !f.home
4254 .join("recovery-lineages")
4255 .join(format!("{}.json", expected.id))
4256 .exists()
4257 );
4258 assert!(
4260 commit_retirement(
4261 &f.home,
4262 &queue,
4263 &selected,
4264 &mut empty,
4265 &source.id,
4266 &selected.runs,
4267 Fx::now()
4268 )
4269 .unwrap_err()
4270 .to_string()
4271 .contains("selected this recovery source")
4272 );
4273 }
4274
4275 #[tokio::test]
4276 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4277 let f = Fx::new().await;
4278 let state = f.run("cccc", false).await;
4279 let wt = &state.candidates[0].worktree;
4280 Fx::target(wt);
4281 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4282 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4283 assert!(
4284 !declared_outputs(&wt.join("target"))
4285 .unwrap()
4286 .contains(&wt.join("target/debug/link"))
4287 );
4288 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4289 let cargo = cargo_locks(&wt.join("target")).unwrap();
4290 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4291 assert_eq!(out.freed_bytes, 0);
4292 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4293 drop(cargo);
4294 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4295 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4296 assert_eq!(out.freed_bytes, 0);
4297 drop(driver);
4298 let exclusive = OutputLease::take(wt, true).unwrap();
4299 assert!(
4300 OutputLease::borrow(wt).is_err(),
4301 "resume cannot start a writer during cleanup"
4302 );
4303 drop(exclusive);
4304 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4305 assert_eq!(
4306 std::fs::read(wt.join("target/notes")).unwrap(),
4307 b"user file"
4308 );
4309 }
4310
4311 #[tokio::test]
4312 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4313 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4314 let f = Fx::new().await;
4315 let source = f.run("aa10", true).await;
4316 let empty = f.run("aa11", false).await;
4317 f.task(&[source, empty.clone()]);
4318 let wt = &empty.candidates[0].worktree;
4319 Fx::target(wt);
4320 let path = wt.join("target").join(leftover);
4321 if leftover.ends_with(".log") {
4322 std::fs::write(&path, "keep log\n").unwrap();
4323 } else {
4324 std::fs::create_dir(&path).unwrap();
4326 }
4327 let mut disk = Fx::disk();
4328 disk.auto_fold = true;
4329 let report = sweep(&f.home, &disk, Fx::now()).await;
4330 assert_eq!(report.freed_bytes, 200, "{report:?}");
4331 assert_eq!(report.retired, 0, "{report:?}");
4332 assert!(!wt.join("target/debug/output").exists());
4333 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4334 assert!(path.exists());
4335 if leftover.ends_with(".log") {
4336 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4337 }
4338 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4339 assert!(worktrees(&empty).await.unwrap().contains(wt));
4340 assert!(
4341 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4342 .await
4343 .unwrap()
4344 );
4345 assert!(
4346 RunState::load_under(&empty.id, &f.home)
4347 .unwrap()
4348 .retired_to
4349 .is_none()
4350 );
4351 }
4352 }
4353
4354 #[tokio::test]
4355 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4356 let f = Fx::new().await;
4357 let state = f.run("aa12", false).await;
4358 let wt = &state.candidates[0].worktree;
4359 Fx::target(wt);
4360 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4361 assert_eq!(report.freed_bytes, 200, "{report:?}");
4362 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4363 for name in ["tmp", "magi-tools", "log"] {
4364 let path = wt.join("target").join(name);
4365 std::fs::create_dir(&path).unwrap();
4366 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4367 std::fs::remove_dir(path).unwrap();
4368 }
4369 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4370 }
4371
4372 #[tokio::test]
4373 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4374 for invalid in [
4375 "tag",
4376 "identity",
4377 "tracked",
4378 "profile-file",
4379 "profile-symlink",
4380 "identity-symlink",
4381 ] {
4382 let f = Fx::new().await;
4383 let state = f.run("aa13", false).await;
4384 let wt = &state.candidates[0].worktree;
4385 Fx::target(wt);
4386 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4387 let target = wt.join("target");
4388 match invalid {
4389 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4390 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4391 "tracked" => {
4392 git::git(wt, &["add", "-f", "target/debug/output"])
4393 .await
4394 .unwrap();
4395 }
4396 "profile-file" => {
4397 std::fs::write(target.join("release"), "keep").unwrap();
4398 }
4399 "profile-symlink" => {
4400 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4401 .unwrap();
4402 }
4403 "identity-symlink" => {
4404 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4405 .unwrap();
4406 std::os::unix::fs::symlink(
4407 target.join("saved-info"),
4408 target.join(".rustc_info.json"),
4409 )
4410 .unwrap();
4411 }
4412 _ => unreachable!(),
4413 }
4414 let before = std::fs::read(target.join("debug/output")).unwrap();
4415 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4416 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4417 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4418 assert!(target.join("tmp").is_dir());
4419 }
4420 }
4421
4422 #[tokio::test]
4423 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4424 let f = Fx::new().await;
4425 let source = f.run("0000", true).await;
4426 let empty = f.run("0001", false).await;
4427 let changed = f.run("0002", false).await;
4428 let unknown = f.run("0003", false).await;
4429 let unproven = f.run("0004", false).await;
4430 f.task(&[
4431 source.clone(),
4432 empty.clone(),
4433 changed.clone(),
4434 unknown.clone(),
4435 unproven.clone(),
4436 ]);
4437 for run in [&empty, &changed, &unknown] {
4438 Fx::target(&run.candidates[0].worktree);
4439 }
4440 let reviewer = empty.worktree_root().join("review-1");
4441 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4442 .await
4443 .unwrap();
4444 Fx::target(&reviewer);
4445 let mut disk = Fx::disk();
4446 disk.auto_fold = false;
4447 let report = sweep(&f.home, &disk, Fx::now()).await;
4448 assert_eq!(report.freed_bytes, 800, "{report:?}");
4449 let wt = &empty.candidates[0].worktree;
4450 assert!(wt.join("target/debug/output.d").exists());
4451 assert!(!wt.join("target/debug/output").exists());
4452 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4453 let proof: CargoResidue =
4454 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4455 assert_eq!(
4456 proof.files[Path::new("debug/output.d")],
4457 std::fs::read(wt.join("target/debug/output.d")).unwrap()
4458 );
4459 std::fs::write(
4460 changed.candidates[0].worktree.join("target/debug/output.d"),
4461 "owner notes",
4462 )
4463 .unwrap();
4464 std::fs::write(
4465 unknown.candidates[0].worktree.join("target/debug/notes"),
4466 "keep",
4467 )
4468 .unwrap();
4469 Fx::target(&unproven.candidates[0].worktree);
4470 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4471 disk.auto_fold = true;
4472 let locks = cargo_locks(&wt.join("target")).unwrap();
4473 let mut report = Report::default();
4474 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4475 assert_eq!(report.retired, 0, "{report:?}");
4476 assert!(
4477 report
4478 .retained
4479 .iter()
4480 .any(|reason| reason.contains("Cargo build"))
4481 );
4482 drop(locks);
4483 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4484 assert_eq!(report.retired, 1, "{report:?}");
4485 assert!(!wt.exists());
4486 assert!(!reviewer.exists());
4487 assert!(
4488 residue_proof(&f.home, &empty.id, &reviewer)
4489 .unwrap()
4490 .exists()
4491 );
4492 assert!(
4493 proof_path.exists(),
4494 "metadata evidence outlives the worktree"
4495 );
4496 assert_eq!(
4497 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4498 Some(source.id)
4499 );
4500 for run in [&changed, &unknown, &unproven] {
4501 assert!(run.candidates[0].worktree.exists());
4502 assert!(
4503 RunState::load_under(&run.id, &f.home)
4504 .unwrap()
4505 .retired_to
4506 .is_none()
4507 );
4508 }
4509 }
4510
4511 #[tokio::test]
4512 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4513 let f = Fx::new().await;
4514 let source = f.run("0000", true).await;
4515 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4516 .await
4517 .unwrap();
4518 let mut runs = vec![source.clone()];
4519 for n in 1..=12 {
4520 runs.push(f.run(&format!("{n:04}"), false).await);
4521 }
4522 let dirty = runs[1].candidates[0].worktree.clone();
4523 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4524 let ignored = runs[2].candidates[0].worktree.clone();
4525 std::fs::create_dir(ignored.join("scratch")).unwrap();
4526 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4527 runs[1].candidates.clear();
4530 runs[1].save_under(&f.home).unwrap();
4531 let task = f.task(&runs);
4532 let mut total = 0;
4533 for _ in 0..4 {
4534 let mut report = Report::default();
4535 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4536 assert!(report.retired <= 5);
4537 total += report.retired;
4538 }
4539 assert_eq!(total, 10);
4540 assert!(dirty.exists());
4541 assert!(ignored.exists());
4542 assert_eq!(
4543 git::rev_parse(&f.repo, &source.candidates[0].branch)
4544 .await
4545 .unwrap(),
4546 source_head
4547 );
4548 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4549 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4550 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4551 let mut released = task.clone();
4552 released.release();
4553 assert_eq!(
4554 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4555 RunState::load_under(id, &f.home)
4556 }),
4557 Some(source.id.clone())
4558 );
4559
4560 assert!(
4561 f.home
4562 .join("runs")
4563 .join(&retired.id)
4564 .join("run.json")
4565 .exists()
4566 );
4567 assert!(
4568 f.home
4569 .join("recovery-lineages")
4570 .join(format!("{}.json", task.id))
4571 .exists()
4572 );
4573 }
4574
4575 #[tokio::test]
4576 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4577 let f = Fx::new().await;
4578 let a = f.run("dddd", true).await;
4579 let b = f.run("eeee", true).await;
4580 let empty = f.run("ffff", false).await;
4581 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4582 assert!(
4583 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4584 "unfinished queue history protects older source branches from ordinary folding"
4585 );
4586 let mut prior = a.clone();
4587 assert!(
4588 crate::graph::fold_run(&mut prior, true, &f.home)
4589 .await
4590 .unwrap_err()
4591 .to_string()
4592 .contains(&task.id)
4593 );
4594 let mut report = Report::default();
4595 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4596 assert_eq!(report.retired, 0);
4597 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4598 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4599 Fx::target(&a.candidates[0].worktree);
4600 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4601 assert!(
4602 report.freed_bytes > 0,
4603 "source retention does not retain regenerable outputs"
4604 );
4605 assert!(a.candidates[0].worktree.exists());
4606 assert!(b.candidates[0].worktree.exists());
4607 assert!(empty.candidates[0].worktree.exists());
4608 }
4609
4610 #[tokio::test]
4611 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4612 let f = Fx::new().await;
4613 let authority = f.run("p000", true).await;
4614 let question_run = f.run("p001", false).await;
4615 let mut checkpoint = f.run("p002", false).await;
4616 checkpoint.land_approval = Some(crate::run::LandApproval {
4617 question: "pending".into(),
4618 head: f.base.clone(),
4619 });
4620 checkpoint.save_under(&f.home).unwrap();
4621 let retained = f.run("p003", false).await;
4622 std::fs::write(
4623 f.home
4624 .join("runs")
4625 .join(&retained.id)
4626 .join("retain-recovery-sources"),
4627 "owner",
4628 )
4629 .unwrap();
4630 let mut partial = f.run("p004", false).await;
4631 let reviewer = partial.worktree_root().join("review-1");
4632 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4633 .await
4634 .unwrap();
4635 partial.retired_to = Some(authority.id.clone());
4637 partial.save_under(&f.home).unwrap();
4638 assert!(
4639 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4640 .await
4641 .unwrap()
4642 );
4643 let git_checkpoint = f.run("p005", false).await;
4644 let location = git::git(
4645 &git_checkpoint.candidates[0].worktree,
4646 &[
4647 "rev-parse",
4648 "--path-format=absolute",
4649 "--git-path",
4650 "sequencer",
4651 ],
4652 )
4653 .await
4654 .unwrap();
4655 std::fs::create_dir(location.trim()).unwrap();
4656 let mut preservation = f.run("p006", false).await;
4657 preservation
4658 .candidate_preservation
4659 .insert('A', "pending preservation".into());
4660 preservation.save_under(&f.home).unwrap();
4661 let task = f.task(&[
4662 preservation.clone(),
4663 git_checkpoint.clone(),
4664 authority.clone(),
4665 question_run.clone(),
4666 checkpoint.clone(),
4667 retained.clone(),
4668 partial.clone(),
4669 ]);
4670 let questions = crate::ask::Questions::at(f.home.join("questions"));
4671 let mut question = crate::ask::Question::new(
4672 question_run.id.clone(),
4673 "implement".into(),
4674 "fixture".into(),
4675 "owner decision".into(),
4676 String::new(),
4677 vec!["keep".into()],
4678 );
4679 questions.put(&mut question).unwrap();
4680 let mut report = Report::default();
4681 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4682 assert_eq!(report.retired, 1, "{report:?}");
4683 assert!(!reviewer.exists());
4684 for state in [
4685 &question_run,
4686 &checkpoint,
4687 &retained,
4688 &git_checkpoint,
4689 &preservation,
4690 ] {
4691 assert!(state.candidates[0].worktree.exists());
4692 assert!(
4693 RunState::load_under(&state.id, &f.home)
4694 .unwrap()
4695 .retired_to
4696 .is_none()
4697 );
4698 }
4699 assert!(questions.get(&question.id).unwrap().status.open());
4700 assert_eq!(
4701 crate::queue::Queue::at(f.home.join("queue"))
4702 .get(&task.id)
4703 .unwrap()
4704 .status,
4705 task.status
4706 );
4707 }
4708
4709 #[tokio::test]
4710 async fn preservation_schema_migration_remains_distinct_from_retirement() {
4711 let f = Fx::new().await;
4712 let mut state = f.run("p007", false).await;
4713 state.schema = 15;
4714 state.save_under(&f.home).unwrap();
4715 let native = RunState::load_under(&state.id, &f.home).unwrap();
4716 assert_eq!(native.schema, crate::run::SCHEMA);
4717 assert!(native.candidate_preservation.is_empty());
4718 assert!(native.retired_to.is_none());
4719 state.schema = 14;
4720 state.save_under(&f.home).unwrap();
4721 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4722 assert!(legacy.candidate_preservation.contains_key(&'A'));
4723 legacy.save_under(&f.home).unwrap();
4724 let restored = RunState::load_under(&state.id, &f.home).unwrap();
4725 assert_eq!(
4726 restored.candidate_preservation,
4727 legacy.candidate_preservation
4728 );
4729 }
4730
4731 #[tokio::test]
4732 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4733 let f = Fx::new().await;
4734 let cache = f.repo.join("cache");
4735 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4736 std::fs::create_dir_all(&cache).unwrap();
4737 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4738 assert!(
4739 crate::cache::maintenance_prune(&f.home, &cache, 1)
4740 .unwrap()
4741 .is_none()
4742 );
4743 drop(guard);
4744 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4745 }
4746
4747 #[tokio::test]
4748 async fn launcher_death_does_not_release_live_tool_output_reservations() {
4749 let f = Fx::new().await;
4750 let run = f.run("abcd", false).await;
4751 let wt = &run.candidates[0].worktree;
4752 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4753 helper
4754 .quiet()
4755 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4756 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4757 .env("MAGI_TEST_WORKTREE", wt)
4758 .stdin(std::process::Stdio::null())
4759 .stdout(std::process::Stdio::null())
4760 .stderr(std::process::Stdio::null())
4761 .kill_on_drop(true);
4762 assert!(
4763 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4764 .await
4765 .unwrap()
4766 .unwrap()
4767 .success()
4768 );
4769 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4770 .unwrap()
4771 .trim()
4772 .parse()
4773 .unwrap();
4774 struct KillTool(i32);
4775 impl Drop for KillTool {
4776 fn drop(&mut self) {
4777 unsafe { libc::kill(self.0, libc::SIGKILL) };
4779 }
4780 }
4781 let tool = KillTool(pid);
4782 assert!(OutputLease::take(wt, true).is_err());
4783 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4784 drop(tool);
4785 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4786 loop {
4787 if OutputLease::take(wt, true).is_ok()
4788 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4789 {
4790 break;
4791 }
4792 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4793 }
4794 })
4795 .await
4796 .unwrap();
4797 }
4798
4799 #[tokio::test]
4800 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4801 let f = Fx::new().await;
4802 let run = f.run("abcd", false).await;
4803 let wt = &run.candidates[0].worktree;
4804 let cache = f.repo.join("cache");
4805 std::fs::create_dir(&cache).unwrap();
4806 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4807 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4808 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4809 cmd.quiet()
4810 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4811 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4812 .stdin(std::process::Stdio::null())
4813 .stdout(std::process::Stdio::null())
4814 .stderr(std::process::Stdio::null())
4815 .kill_on_drop(true);
4816 if clear_owner {
4817 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4818 } else {
4819 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4820 }
4821 let gib = 1024 * 1024 * 1024;
4822 let mut leases = vec![
4823 OutputLease::borrow(wt).unwrap().unwrap(),
4824 OutputLease::cache(&f.home, &cache, false).unwrap(),
4825 ];
4826 leases.extend(
4827 OutputLease::admit_with(
4828 &f.home,
4829 std::slice::from_ref(wt),
4830 8 * gib,
4831 |_| Ok(9 * gib),
4832 )
4833 .unwrap(),
4834 );
4835 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4836 assert!(
4837 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4838 .await
4839 .unwrap()
4840 .unwrap()
4841 .success()
4842 );
4843 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4844 struct KillTool(Option<i32>);
4845 impl Drop for KillTool {
4846 fn drop(&mut self) {
4847 if let Some(pid) = self.0 {
4849 unsafe { libc::kill(pid, libc::SIGKILL) };
4850 }
4851 }
4852 }
4853 let mut tool = KillTool(Some(pid));
4854 assert!(OutputLease::take(wt, true).is_err());
4858 let finish = if cancel {
4859 drop(running);
4860 None
4861 } else {
4862 Some(tokio::spawn(async move { running.finish().await }))
4863 };
4864 if clear_owner {
4865 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4866 assert!(crate::proc::pid_alive(pid as u32));
4867 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4868 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4869 assert!(
4870 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4871 9 * gib
4872 ))
4873 .is_err(),
4874 "orphan still owns its disk reservation"
4875 );
4876 assert!(
4877 crate::cache::maintenance_prune(&f.home, &cache, 0)
4878 .unwrap()
4879 .is_none()
4880 );
4881 if let Some(finish) = &finish {
4882 assert!(!finish.is_finished());
4883 }
4884 unsafe { libc::kill(pid, libc::SIGKILL) };
4886 }
4887 if let Some(finish) = finish {
4888 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4889 .await
4890 .unwrap()
4891 .unwrap();
4892 }
4893 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4894 loop {
4895 if OutputLease::take(wt, true).is_ok()
4896 && OutputLease::cache(&f.home, &cache, true).is_ok()
4897 && OutputLease::admit_with(
4898 &f.home,
4899 std::slice::from_ref(wt),
4900 8 * gib,
4901 |_| Ok(9 * gib),
4902 )
4903 .is_ok()
4904 {
4905 if !clear_owner {
4907 assert!(!crate::proc::pid_alive(pid as u32));
4908 }
4909 break;
4910 }
4911 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4912 }
4913 })
4914 .await
4915 .unwrap();
4916 tool.0 = None;
4917 }
4918 }
4919
4920 #[tokio::test]
4921 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4922 let f = Fx::new().await;
4923 let run = f.run("abcd", false).await;
4924 let wt = &run.candidates[0].worktree;
4925 let cache = f.repo.join("cache");
4926 std::fs::create_dir(&cache).unwrap();
4927 let pid_file = f.repo.join("child-pid");
4928 let mut cmd = tokio::process::Command::new("sh");
4929 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4930 .quiet()
4931 .env("CHILD_PID_FILE", &pid_file)
4932 .process_group(0)
4933 .kill_on_drop(true);
4934 let running = crate::seat_process::Running::spawn(
4935 cmd,
4936 None,
4937 vec![
4938 OutputLease::borrow(wt).unwrap().unwrap(),
4939 OutputLease::cache(&f.home, &cache, false).unwrap(),
4940 ],
4941 )
4942 .unwrap();
4943 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4944 while !pid_file.exists() {
4945 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4946 }
4947 })
4948 .await
4949 .unwrap();
4950 assert!(OutputLease::take(wt, true).is_err());
4951 assert!(
4952 crate::cache::maintenance_prune(&f.home, &cache, 0)
4953 .unwrap()
4954 .is_none()
4955 );
4956 let pid: u32 = std::fs::read_to_string(pid_file)
4957 .unwrap()
4958 .trim()
4959 .parse()
4960 .unwrap();
4961 drop(running);
4962 tokio::time::timeout(std::time::Duration::from_secs(10), async {
4963 loop {
4964 if let Ok(guard) = OutputLease::take(wt, true) {
4965 assert!(
4966 !crate::proc::pid_alive(pid),
4967 "output lock released before tool death"
4968 );
4969 drop(guard);
4970 break;
4971 }
4972 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4973 }
4974 })
4975 .await
4976 .unwrap();
4977 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4978 }
4979
4980 #[tokio::test]
4981 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
4982 let f = Fx::new().await;
4983 let reason = tokio::time::timeout(
4984 std::time::Duration::from_secs(2),
4985 emergency(vec![f.repo.clone()], u64::MAX),
4986 )
4987 .await
4988 .unwrap();
4989 assert!(reason.contains("disk pressure"));
4990 assert!(
4991 tokio::time::timeout(
4992 std::time::Duration::from_millis(10),
4993 emergency(vec![f.repo.clone()], 0)
4994 )
4995 .await
4996 .is_err()
4997 );
4998 assert_eq!(
4999 std::fs::read_to_string(f.repo.join("source")).unwrap(),
5000 "base\n"
5001 );
5002 }
5003
5004 #[tokio::test]
5005 async fn automatic_requeues_cannot_refund_failed_input_generation() {
5006 let f = Fx::new().await;
5007 let mut run = f.run("1234", false).await;
5008 run.prep_failure = Some("cherry-pick conflict".into());
5009 let mut task = f.task(std::slice::from_ref(&run));
5010 let inputs = crate::queue::recovery_inputs(&run.sources);
5011 for _ in 0..108 {
5012 task.recovery_failed(&run);
5013 task.requeue();
5014 assert_eq!(task.attempts, 0);
5015 }
5016 let bytes = serde_json::to_vec(&task).unwrap();
5017 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
5018 assert!(
5019 restored
5020 .recovery_exhausted(&run.base_commit, &inputs, 2)
5021 .is_some()
5022 );
5023 assert!(
5024 restored
5025 .recovery_exhausted("different-base", &inputs, 2)
5026 .is_none()
5027 );
5028 let original_base = run.base_commit.clone();
5029 run.base_commit = "different-base".into();
5030 restored.recovery_failed(&run);
5031 assert!(
5032 restored
5033 .recovery_exhausted(&original_base, &inputs, 2)
5034 .is_some(),
5035 "returning to old inputs does not replenish their budget"
5036 );
5037 assert!(
5038 restored
5039 .recovery_exhausted(&run.base_commit, &inputs, 2)
5040 .is_none()
5041 );
5042 restored.release_operator("test-owner", None);
5043 assert!(
5044 restored
5045 .recovery_exhausted(&run.base_commit, &inputs, 2)
5046 .is_none()
5047 );
5048 }
5049 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5050 let mut state = f.run(tag, useful).await;
5051 state.status = RunStatus::Superseded;
5052 state.save_under(&f.home).unwrap();
5053 state
5054 }
5055
5056 fn historical_review(state: &mut RunState, head: &str) {
5057 state.reviews.push(
5058 serde_json::from_value(serde_json::json!({
5059 "round": state.reviews.len() + 1, "head": head, "reviews": []
5060 }))
5061 .unwrap(),
5062 );
5063 }
5064
5065 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5066 let path = state.worktree_root().join("review-1");
5067 git::worktree_add_detached(&f.repo, &path, head)
5068 .await
5069 .unwrap();
5070 path
5071 }
5072
5073 #[tokio::test]
5074 async fn source_eviction_separates_run_authority_from_read_seats() {
5075 for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5076 let f = Fx::new().await;
5077 let mut state = inactive_source(&f, "ef01", true).await;
5078 state.status = status;
5079 state.pr = Some(
5080 serde_json::from_value(serde_json::json!({
5081 "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5082 "checks":"green", "round":1, "rounds":1
5083 }))
5084 .unwrap(),
5085 );
5086 state.pr_head = Some(state.candidates[0].branch.clone());
5087 state.pr_merged_head = Some(f.base.clone());
5088 state.recovery = Some(
5089 serde_json::from_value(serde_json::json!({
5090 "from_run": "20260101-000000-abcd", "generation": 1,
5091 "mode": "full-review"
5092 }))
5093 .unwrap(),
5094 );
5095 state.pr_recovery = Some(
5096 serde_json::from_value(serde_json::json!({
5097 "url": "https://example.invalid/pull/1", "branch": "public",
5098 "head": f.base, "source_run": state.id,
5099 "instruction": "keep implementation", "approved_at": Fx::now()
5100 }))
5101 .unwrap(),
5102 );
5103 state.save_under(&f.home).unwrap();
5104 let review = reviewer(&f, &state, &f.base).await;
5105 let mut other = inactive_source(&f, "ef02", true).await;
5106 other.pr_recovery = state.pr_recovery.clone();
5107 other.save_under(&f.home).unwrap();
5108 let mut task = f.task(&[state.clone(), other]);
5109 task.pr_recovery = state.pr_recovery.clone();
5110 crate::queue::Queue::at(f.home.join("queue"))
5111 .put(&mut task)
5112 .unwrap();
5113 let run_path = f.home.join("runs").join(&state.id).join("run.json");
5114 let before = std::fs::read(&run_path).unwrap();
5115 let task_before = serde_json::to_value(&task).unwrap();
5116 let mut report = Report::default();
5117 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5118 assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5119 assert!(!review.exists());
5120 assert!(state.candidates[0].worktree.exists());
5121 assert!(
5122 git::branch_exists(&f.repo, &state.candidates[0].branch)
5123 .await
5124 .unwrap()
5125 );
5126 assert_eq!(std::fs::read(&run_path).unwrap(), before);
5127 assert_eq!(
5128 serde_json::to_value(
5129 crate::queue::Queue::at(f.home.join("queue"))
5130 .get(&task.id)
5131 .unwrap()
5132 )
5133 .unwrap(),
5134 task_before
5135 );
5136 }
5137 }
5138
5139 #[tokio::test]
5140 async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5141 for kind in ["own-path", "active-seat", "paused-session"] {
5142 let f = Fx::new().await;
5143 let mut state = inactive_source(&f, "ef03", true).await;
5144 let review = reviewer(&f, &state, &f.base).await;
5145 let expected = match kind {
5146 "own-path" => {
5147 state.attachments.push(review.join("source"));
5148 "run.attachments[0]"
5149 }
5150 "active-seat" => {
5151 state.active.insert(
5152 "review-1".into(),
5153 serde_json::from_value(serde_json::json!({
5154 "node":"review", "started_at":Fx::now(), "timeout_secs":60
5155 }))
5156 .unwrap(),
5157 );
5158 "continuation lease"
5159 }
5160 _ => {
5161 state.status = RunStatus::Blocked;
5162 state.parked = true;
5163 let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5164 seat.turns = 2;
5165 seat.captured_session = Some("saved-session".into());
5166 state.seats.insert("review-1".into(), seat);
5167 "resume contract"
5168 }
5169 };
5170 state.save_under(&f.home).unwrap();
5171 let mut report = Report::default();
5172 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5173 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5174 assert!(review.exists());
5175 assert!(
5176 report.retained.iter().any(|r| r.contains(expected)),
5177 "{kind}: {report:?}"
5178 );
5179 restore_execution_seats(&f.home, &state).await.unwrap();
5182 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5183 }
5184 }
5185
5186 #[tokio::test]
5187 async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5188 for kind in ["waiting", "lease", "historical", "unknown"] {
5189 let f = Fx::new().await;
5190 let state = inactive_source(&f, "ef06", true).await;
5191 let review = reviewer(&f, &state, &f.base).await;
5192 let store = crate::ask::Questions::at(f.home.join("questions"));
5193 let mut question = crate::ask::Question::new(
5194 "20260101-000000-ffff".into(),
5195 "review".into(),
5196 "review-1".into(),
5197 "fixture".into(),
5198 "fixture".into(),
5199 Vec::new(),
5200 );
5201 question.cwd = Some(review.to_string_lossy().into_owned());
5202 if kind != "waiting" {
5203 question.status = crate::ask::QuestionStatus::Abandoned;
5204 }
5205 store.put(&mut question).unwrap();
5206 if kind == "lease" {
5207 store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5208 }
5209 if kind == "unknown" {
5210 let path = store.path_of(&question.id);
5211 let mut raw: serde_json::Value =
5212 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5213 raw["future_cwd_authority"] = serde_json::Value::Null;
5214 std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5215 }
5216 let mut report = Report::default();
5217 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5218 assert_eq!(
5219 report.evicted_copies,
5220 usize::from(kind == "historical"),
5221 "{kind}: {report:?}"
5222 );
5223 assert_eq!(review.exists(), kind != "historical");
5224 if kind == "waiting" || kind == "lease" {
5225 assert!(
5226 report
5227 .retained
5228 .iter()
5229 .any(|r| r.contains("question") && r.contains("cwd")),
5230 "{report:?}"
5231 );
5232 }
5233 }
5234 }
5235
5236 #[tokio::test]
5237 async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5238 let f = Fx::new().await;
5239 let mut state = inactive_source(&f, "ef07", true).await;
5240 let review = reviewer(&f, &state, &f.base).await;
5241 let mut report = Report::default();
5242 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5243 assert_eq!(report.evicted_copies, 1, "{report:?}");
5244 state.config.worktree.setup = vec![crate::config::SetupStep {
5245 run: Some("printf preserved > source; exit 1".into()),
5246 ..Default::default()
5247 }];
5248 state.save_under(&f.home).unwrap();
5249 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5250 let marker = f
5251 .home
5252 .join("runs")
5253 .join(&state.id)
5254 .join("artifacts/restore-review-1.pending");
5255 assert!(marker.exists());
5256 assert_eq!(
5257 std::fs::read_to_string(review.join("source")).unwrap(),
5258 "preserved"
5259 );
5260 assert!(
5261 restore_execution_seats(&f.home, &state)
5262 .await
5263 .unwrap_err()
5264 .to_string()
5265 .contains("interrupted execution restoration")
5266 );
5267 let mut retry = Report::default();
5268 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5269 assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5270 assert!(
5271 retry
5272 .retained
5273 .iter()
5274 .any(|r| r.contains("restoration checkpoint")),
5275 "{retry:?}"
5276 );
5277 assert!(marker.exists());
5278 assert_eq!(
5279 std::fs::read_to_string(review.join("source")).unwrap(),
5280 "preserved"
5281 );
5282 }
5283
5284 #[tokio::test]
5285 async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5286 for kind in [
5287 "alternates",
5288 "promisor-config",
5289 "promisor-pack",
5290 "objects-symlink",
5291 ] {
5292 let f = Fx::new().await;
5293 let state = inactive_source(&f, "ef11", true).await;
5294 let review = reviewer(&f, &state, &f.base).await;
5295 let common = crate::repo_lock::common_dir(&f.repo)
5296 .await
5297 .unwrap()
5298 .unwrap();
5299 match kind {
5300 "alternates" => {
5301 let borrowed = f._temp.path().join("borrowed-objects");
5302 std::fs::create_dir(&borrowed).unwrap();
5303 std::fs::write(
5304 common.join("objects/info/alternates"),
5305 format!("{}\n", borrowed.display()),
5306 )
5307 .unwrap();
5308 }
5309 "promisor-config" => {
5310 git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5311 .await
5312 .unwrap();
5313 }
5314 "promisor-pack" => {
5315 std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5316 }
5317 _ => {
5318 let borrowed = f._temp.path().join("borrowed-objects");
5319 std::fs::rename(common.join("objects"), &borrowed).unwrap();
5320 std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5321 }
5322 }
5323 let mut report = Report::default();
5324 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5325 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5326 assert!(review.exists());
5327 assert!(
5328 report.retained.iter().any(|r| r.contains("source anchor")),
5329 "{kind}: {report:?}"
5330 );
5331 }
5332 }
5333
5334 #[tokio::test]
5335 async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5336 let mut f = Fx::new().await;
5337 crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5338 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5339 assert_eq!(
5340 git::git(&f.repo, &["rev-list", "--count", &f.base])
5341 .await
5342 .unwrap(),
5343 "5001"
5344 );
5345 let state = inactive_source(&f, "ef10", true).await;
5346 let review = reviewer(&f, &state, &f.base).await;
5347 std::fs::write(review.join("source"), "bounded review delta").unwrap();
5348 git::git(&review, &["add", "."]).await.unwrap();
5349 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5350 let head = git::git(
5351 &review,
5352 &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5353 )
5354 .await
5355 .unwrap();
5356 git::git(&review, &["checkout", "--detach", &head])
5357 .await
5358 .unwrap();
5359 let run_file = f.home.join("runs").join(&state.id).join("run.json");
5360 let run_before = std::fs::read(&run_file).unwrap();
5361 let mut report = Report::default();
5362 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5363 assert_eq!(report.evicted_copies, 1, "{report:?}");
5364 assert!(!review.exists());
5365 assert!(state.candidates[0].worktree.exists());
5366 assert!(
5367 git::branch_exists(&f.repo, &state.candidates[0].branch)
5368 .await
5369 .unwrap()
5370 );
5371 assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5372 let archive = f
5373 .home
5374 .join("runs")
5375 .join(&state.id)
5376 .join("artifacts/source-evidence")
5377 .join(format!("{head}-{}", f.base));
5378 let manifest: serde_json::Value =
5379 serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5380 assert_eq!(manifest["anchor"], f.base);
5381 assert!(
5382 std::fs::metadata(archive.join("objects.pack"))
5383 .unwrap()
5384 .len()
5385 < 16 * 1024
5386 );
5387 let pin = format!("refs/magi/source-anchors/{}", f.base);
5388 assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5389 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5390 .await
5391 .unwrap();
5392 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5393 assert!(
5394 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5395 .await
5396 .unwrap()
5397 .ok()
5398 );
5399 let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5400 .await
5401 .unwrap();
5402 for kind in ["missing", "moved", "symbolic", "symlink"] {
5404 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5405 .await
5406 .unwrap();
5407 match kind {
5408 "moved" => {
5409 git::git(&f.repo, &["update-ref", &pin, &other])
5410 .await
5411 .unwrap();
5412 }
5413 "symbolic" => {
5414 git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5415 .await
5416 .unwrap();
5417 }
5418 "symlink" => {
5419 let common = crate::repo_lock::common_dir(&f.repo)
5420 .await
5421 .unwrap()
5422 .unwrap();
5423 let target = f._temp.path().join("operator-anchor");
5424 std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5425 let file = common.join(&pin);
5426 std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5427 std::os::unix::fs::symlink(target, file).unwrap();
5428 }
5429 _ => {}
5430 }
5431 assert!(
5432 restore_execution_seats(&f.home, &state).await.is_err(),
5433 "{kind}"
5434 );
5435 assert!(!review.exists());
5436 assert!(
5437 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5438 .await
5439 .unwrap()
5440 .ok()
5441 );
5442 if kind == "symlink" {
5443 let common = crate::repo_lock::common_dir(&f.repo)
5444 .await
5445 .unwrap()
5446 .unwrap();
5447 std::fs::remove_file(common.join(&pin)).unwrap();
5448 } else {
5449 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5450 .await
5451 .unwrap();
5452 }
5453 git::git(&f.repo, &["update-ref", &pin, &f.base])
5454 .await
5455 .unwrap();
5456 }
5457 assert!(
5460 git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5461 .await
5462 .unwrap()
5463 );
5464 git::git(
5465 &f.repo,
5466 &[
5467 "update-ref",
5468 "-d",
5469 &format!("refs/heads/{}", state.candidates[0].branch),
5470 ],
5471 )
5472 .await
5473 .unwrap();
5474 git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5475 .await
5476 .unwrap();
5477 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5478 .await
5479 .unwrap();
5480 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5481 assert_eq!(
5482 git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5483 .await
5484 .unwrap(),
5485 pin
5486 );
5487 assert_eq!(
5488 git::git(&f.repo, &["rev-list", "--count", &f.base])
5489 .await
5490 .unwrap(),
5491 "5001"
5492 );
5493 restore_execution_seats(&f.home, &state).await.unwrap();
5494 assert_eq!(
5495 git::git(&review, &["rev-list", "--count", "HEAD"])
5496 .await
5497 .unwrap(),
5498 "5002"
5499 );
5500 assert_eq!(
5501 git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5502 .await
5503 .unwrap(),
5504 "false"
5505 );
5506 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5507 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5508 let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5509 assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5510 std::fs::remove_dir_all(&f.repo).unwrap();
5511 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5512 let inspection = f._temp.path().join("shared-history-inspection");
5513 restore_seat(&f.home, &state.id, &head, &inspection)
5514 .await
5515 .unwrap();
5516 assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5517 assert_eq!(
5518 git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5519 .await
5520 .unwrap(),
5521 "true"
5522 );
5523 assert_eq!(
5524 git::git(&inspection, &["log", "--format=%H"])
5525 .await
5526 .unwrap(),
5527 format!("{head}\n{}", f.base)
5528 );
5529 assert!(!inspection.join(".git/objects/info/alternates").exists());
5530 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5531 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5532 }
5533
5534 #[tokio::test]
5535 async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5536 let f = Fx::new().await;
5537 let state = inactive_source(&f, "ef08", true).await;
5538 let review = reviewer(&f, &state, &f.base).await;
5539 let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5540 let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5541 .await
5542 .unwrap();
5543 let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
5544 .await
5545 .unwrap();
5546 git::git(&review, &["checkout", "--detach", &head])
5547 .await
5548 .unwrap();
5549 let archive =
5550 crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
5551 let mut report = Report::default();
5552 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5553 assert_eq!(report.evicted_copies, 0, "{report:?}");
5554 assert!(review.exists());
5555 assert!(
5556 report
5557 .retained
5558 .iter()
5559 .any(|r| r.contains("incomplete archived commit ancestry")),
5560 "{report:?}"
5561 );
5562 assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
5564 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5565 .await
5566 .unwrap();
5567 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5568 for oid in [&parent, &head] {
5569 assert!(
5570 !git::git_raw(&f.repo, &["cat-file", "-e", oid])
5571 .await
5572 .unwrap()
5573 .ok()
5574 );
5575 }
5576 let patch = source_diff(&archive).await.unwrap();
5577 assert!(
5578 crate::fold_evidence::import_snapshot(&archive, &f.repo)
5579 .await
5580 .is_err()
5581 );
5582 assert!(
5583 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5584 .await
5585 .unwrap()
5586 .ok()
5587 );
5588 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5589 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5590 let inspection = f._temp.path().join("legacy-tree-inspection");
5591 restore_source(&archive, &inspection).await.unwrap();
5592 assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
5593 assert!(
5594 !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
5595 .await
5596 .unwrap()
5597 .ok()
5598 );
5599 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5600 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5601 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5602 assert!(state.candidates[0].worktree.exists());
5603 }
5604
5605 #[tokio::test]
5606 async fn source_eviction_retains_ancestry_above_expanded_budget() {
5607 let f = Fx::new().await;
5608 let state = inactive_source(&f, "ef09", true).await;
5609 let review = reviewer(&f, &state, &f.base).await;
5610 std::fs::File::create(review.join("ancestor-only"))
5611 .unwrap()
5612 .set_len(64 * 1024 * 1024 + 1)
5613 .unwrap();
5614 git::git(&review, &["add", "."]).await.unwrap();
5615 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5616 let parent = git::git(
5617 &review,
5618 &["commit-tree", &parent_tree, "-m", "large ancestor"],
5619 )
5620 .await
5621 .unwrap();
5622 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5623 git::git(&review, &["add", "."]).await.unwrap();
5624 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5625 let head = git::git(
5626 &review,
5627 &[
5628 "commit-tree",
5629 &tree,
5630 "-p",
5631 &parent,
5632 "-m",
5633 "small final tree",
5634 ],
5635 )
5636 .await
5637 .unwrap();
5638 git::git(&review, &["checkout", "--detach", &head])
5639 .await
5640 .unwrap();
5641 let mut report = Report::default();
5642 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5643 assert_eq!(report.evicted_copies, 0, "{report:?}");
5644 assert!(
5645 report
5646 .retained
5647 .iter()
5648 .any(|r| r.contains("expanded object limit exceeded")),
5649 "{report:?}"
5650 );
5651 assert!(review.exists());
5652 assert!(state.candidates[0].worktree.exists());
5653 }
5654
5655 #[tokio::test]
5656 async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
5657 let f = Fx::new().await;
5658 let state = inactive_source(&f, "ef04", true).await;
5659 let review = reviewer(&f, &state, &f.base).await;
5660 std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
5661 git::git(&review, &["add", "."]).await.unwrap();
5662 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5663 let parent = git::git(
5664 &review,
5665 &["commit-tree", &parent_tree, "-m", "unrelated parent"],
5666 )
5667 .await
5668 .unwrap();
5669 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5670 std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
5671 git::git(&review, &["add", "."]).await.unwrap();
5672 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5673 let head = git::git(
5674 &review,
5675 &[
5676 "commit-tree",
5677 &tree,
5678 "-p",
5679 &parent,
5680 "-m",
5681 "unrelated review checkpoint",
5682 ],
5683 )
5684 .await
5685 .unwrap();
5686 git::git(&review, &["checkout", "--detach", &head])
5687 .await
5688 .unwrap();
5689 let mut report = Report::default();
5690 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5691 assert_eq!(report.evicted_copies, 1, "{report:?}");
5692 let ledger = f
5693 .home
5694 .join("runs")
5695 .join(&state.id)
5696 .join("artifacts/eviction-review-1.json");
5697 let original = std::fs::read(&ledger).unwrap();
5698 let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
5699 assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
5700 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5701 .await
5702 .unwrap();
5703 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5704 assert!(
5705 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5706 .await
5707 .unwrap()
5708 .ok()
5709 );
5710 assert!(
5711 !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
5712 .await
5713 .unwrap()
5714 .ok()
5715 );
5716 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5717 assert!(
5718 seat_heads(&f.home, &state.id)
5719 .await
5720 .unwrap()
5721 .contains_key(&head)
5722 );
5723 restore_execution_seats(&f.home, &state).await.unwrap();
5724 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
5725 assert_eq!(
5726 git::git(&review, &["log", "--format=%H"]).await.unwrap(),
5727 format!("{head}\n{parent}")
5728 );
5729 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5730 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5731 let inspection = f._temp.path().join("complete-history-inspection");
5732 restore_seat(&f.home, &state.id, &head, &inspection)
5733 .await
5734 .unwrap();
5735 assert_eq!(
5736 git::git(&inspection, &["log", "--format=%H"])
5737 .await
5738 .unwrap(),
5739 format!("{head}\n{parent}")
5740 );
5741 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5742 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5743
5744 assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
5745 assert!(
5746 !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
5747 .await
5748 .unwrap()
5749 .ok()
5750 );
5751 assert_eq!(
5752 std::fs::read_to_string(review.join("source")).unwrap(),
5753 "unrelated review checkpoint"
5754 );
5755 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5756 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5757 assert_eq!(
5758 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5759 "keep"
5760 );
5761 std::fs::remove_file(review.join("operator-notes")).unwrap();
5762 restore_execution_seats(&f.home, &state).await.unwrap();
5763 let mut second = Report::default();
5764 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
5765 assert_eq!(second.evicted_copies, 1, "{second:?}");
5766 assert_eq!(std::fs::read(&ledger).unwrap(), original);
5767 assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
5768 let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
5771 legacy["version"] = 1.into();
5772 legacy.as_object_mut().unwrap().remove("base");
5773 std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
5774 let mut advanced = state.clone();
5775 advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5776 .await
5777 .unwrap();
5778 advanced.save_under(&f.home).unwrap();
5779 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5780 let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
5782 let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
5783 let patch = std::fs::read(archive.join("change.patch")).unwrap();
5784 std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
5785 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5786 assert!(!review.exists());
5787 std::fs::write(archive.join("change.patch"), patch).unwrap();
5788 std::fs::create_dir(&review).unwrap();
5789 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5790 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5791 assert_eq!(
5792 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5793 "keep"
5794 );
5795 }
5796
5797 #[tokio::test]
5798 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
5799 use std::os::unix::fs::PermissionsExt;
5800 let mut f = Fx::new().await;
5801 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
5802 git::git(&f.repo, &["add", "."]).await.unwrap();
5803 git::git(&f.repo, &["commit", "-m", "binary base"])
5804 .await
5805 .unwrap();
5806 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5807 let mut state = inactive_source(&f, "ec01", true).await;
5808 let candidate = state.candidates[0].worktree.clone();
5809 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5810 historical_review(&mut state, &earlier);
5811 let review = reviewer(&f, &state, &earlier).await;
5812 std::fs::remove_file(candidate.join("source")).unwrap();
5813 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5814 std::fs::write(candidate.join("script"), "run\n").unwrap();
5815 std::fs::set_permissions(
5816 candidate.join("script"),
5817 std::fs::Permissions::from_mode(0o755),
5818 )
5819 .unwrap();
5820 git::git(&candidate, &["add", "."]).await.unwrap();
5821 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5822 .await
5823 .unwrap();
5824 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5825 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5826 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5827 .await
5828 .unwrap();
5829 historical_review(&mut state, &head);
5830 historical_review(&mut state, &head);
5831 state.status = RunStatus::Superseded;
5832 state.save_under(&f.home).unwrap();
5833 let task = f.task(std::slice::from_ref(&state));
5834 let before = serde_json::to_value(&state).unwrap();
5835 git::git(&review, &["reset", "--hard", &head])
5836 .await
5837 .unwrap();
5838 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5839 assert_eq!(report.evicted_copies, 1, "{report:?}");
5840 assert!(!review.exists());
5841 assert!(candidate.exists(), "selected implementation survives");
5842 assert!(report.source_unlinked_logical_bytes > 0);
5843 assert!(report.source_unlinked_allocated_bytes.is_some());
5844 assert!(report.source_evidence_added_allocated_bytes.is_some());
5845 assert_eq!(
5846 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5847 before
5848 );
5849 assert_eq!(
5850 crate::queue::Queue::at(f.home.join("queue"))
5851 .get(&task.id)
5852 .unwrap()
5853 .runs,
5854 task.runs
5855 );
5856 let archive = f
5857 .home
5858 .join("runs")
5859 .join(&state.id)
5860 .join("artifacts/source-evidence")
5861 .join(format!("{head}-{}", f.base));
5862 let patch = source_diff(&archive).await.unwrap();
5863 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5864 assert!(String::from_utf8_lossy(&patch).contains("100755"));
5865 assert!(
5866 git::worktree_remove_clean(&f.repo, &candidate)
5867 .await
5868 .unwrap()
5869 );
5870 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5871 .await
5872 .unwrap();
5873 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5874 .await
5875 .unwrap();
5876 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5877 assert!(
5878 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5879 .await
5880 .unwrap()
5881 .ok()
5882 );
5883 std::fs::remove_dir_all(&f.repo).unwrap();
5885 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5886 let restored = f._temp.path().join("restored");
5887 restore_source(&archive, &restored).await.unwrap();
5888 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5889 assert_eq!(
5890 std::fs::read(restored.join("binary")).unwrap(),
5891 b"new\0binary\xfe"
5892 );
5893 assert!(!restored.join("source").exists());
5894 assert_ne!(
5895 std::fs::metadata(restored.join("script"))
5896 .unwrap()
5897 .permissions()
5898 .mode()
5899 & 0o111,
5900 0
5901 );
5902 assert!(restore_source(&archive, &restored).await.is_err());
5903 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5906 .await
5907 .unwrap();
5908 let patch_file = f._temp.path().join("saved.patch");
5909 std::fs::write(&patch_file, &patch).unwrap();
5910 git::git(
5911 &restored,
5912 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5913 )
5914 .await
5915 .unwrap();
5916 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5917 assert_eq!(
5918 std::fs::read(restored.join("binary")).unwrap(),
5919 b"new\0binary\xfe"
5920 );
5921 }
5922
5923 #[tokio::test]
5924 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5925 let f = Fx::new().await;
5926 let state = inactive_source(&f, "ec10", true).await;
5927 let review = reviewer(&f, &state, &f.base).await;
5928 let mut successor = inactive_source(&f, "ec11", true).await;
5929 successor.retired_to = Some(state.id.clone());
5930 successor.save_under(&f.home).unwrap();
5931 f.task(&[state.clone(), successor]);
5932
5933 let mut tips = Vec::new();
5934 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5935 std::fs::write(review.join("source"), contents).unwrap();
5936 git::git(&review, &["add", "."]).await.unwrap();
5937 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5938 let tip = git::git(
5939 &review,
5940 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5941 )
5942 .await
5943 .unwrap();
5944 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5945 tips.push(tip);
5946 git::git(&review, &["reset", "--hard", &f.base])
5947 .await
5948 .unwrap();
5949 }
5950 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5952 .await
5953 .unwrap();
5954 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5955 .await
5956 .unwrap();
5957 let mut report = Report::default();
5958 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5959 assert_eq!(report.evicted_copies, 1, "{report:?}");
5960 assert!(!review.exists());
5961 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5962 .await
5963 .unwrap();
5964 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5965 for tip in &tips {
5966 assert!(
5967 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5968 .await
5969 .unwrap()
5970 .ok()
5971 );
5972 }
5973 std::fs::remove_dir_all(&f.repo).unwrap();
5975 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
5976 for (i, tip) in tips.iter().enumerate() {
5977 assert!(
5978 shown
5979 .iter()
5980 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
5981 );
5982 let destination = f._temp.path().join(format!("restore-history-{i}"));
5983 restore_seat(&f.home, &state.id, tip, &destination)
5984 .await
5985 .unwrap();
5986 assert_eq!(
5987 std::fs::read_to_string(destination.join("source")).unwrap(),
5988 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
5989 );
5990 }
5991 assert!(
5992 restore_seat(
5993 &f.home,
5994 &state.id,
5995 "unknown",
5996 &f._temp.path().join("missing")
5997 )
5998 .await
5999 .is_err()
6000 );
6001 }
6002
6003 #[tokio::test]
6004 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
6005 fn corrupt(path: &Path) -> Result<()> {
6006 let root = path.parent().unwrap();
6007 let marker = root.join("archive-to-corrupt");
6010 let archive = std::fs::read_to_string(&marker)?;
6011 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
6012 Ok(())
6013 }
6014 let f = Fx::new().await;
6015 let state = inactive_source(&f, "ec12", true).await;
6016 let review = reviewer(&f, &state, &f.base).await;
6017 let archive = f
6018 .home
6019 .join("runs")
6020 .join(&state.id)
6021 .join("artifacts/source-evidence")
6022 .join(format!("{}-{}", f.base, f.base));
6023 std::fs::write(
6024 state.worktree_root().join("archive-to-corrupt"),
6025 archive.to_str().unwrap(),
6026 )
6027 .unwrap();
6028 let mut report = Report::default();
6029 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
6030 assert_eq!(report.evicted_copies, 0, "{report:?}");
6031 assert!(review.exists());
6032 assert!(
6033 report
6034 .retained
6035 .iter()
6036 .any(|r| r.contains("corrupt snapshot")),
6037 "{report:?}"
6038 );
6039 }
6040
6041 #[tokio::test]
6042 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6043 for kind in [
6044 "dirty",
6045 "hidden",
6046 "ignored",
6047 "empty-directory",
6048 "untracked",
6049 "borrowed",
6050 "borrowed-symlink",
6051 "lease",
6052 "owner",
6053 "driver",
6054 "task-pin",
6055 "task-replacement",
6056 "malformed-orig-head",
6057 "grace",
6058 "resumable",
6059 "archive-symlink",
6060 "root-symlink",
6061 "checkpoint",
6062 "symlink",
6063 "retained",
6064 "unreadable",
6065 "unknown-field",
6066 "unknown-task-field",
6067 ] {
6068 let f = Fx::new().await;
6069 let mut state = inactive_source(&f, "ec02", true).await;
6070 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6071 .await
6072 .unwrap();
6073 historical_review(&mut state, &head);
6074 let review = reviewer(&f, &state, &head).await;
6075 let mut output = None;
6076 let mut owner = None;
6077 let mut driver = None;
6078 let mut disk = Fx::disk();
6079 match kind {
6080 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6081 "hidden" => {
6082 git::git(&review, &["update-index", "--assume-unchanged", "source"])
6083 .await
6084 .unwrap();
6085 std::fs::write(review.join("source"), "notes").unwrap();
6086 }
6087 "ignored" => {
6088 std::fs::create_dir(review.join("scratch")).unwrap();
6089 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6090 }
6091 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6092 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6093 "borrowed" => {
6094 let mut other = f.run("ec03", true).await;
6095 other.candidates[0].worktree = review.clone();
6096 other.save_under(&f.home).unwrap();
6097 }
6098 "borrowed-symlink" => {
6099 let alias = f._temp.path().join("borrowed-alias");
6100 std::os::unix::fs::symlink(&review, &alias).unwrap();
6101 let mut other = f.run("ec03", true).await;
6102 other.candidates[0].worktree = alias;
6103 other.save_under(&f.home).unwrap();
6104 }
6105 "lease" => output = OutputLease::borrow(&review).unwrap(),
6106 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6107 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6108 "task-pin" => {
6109 let mut task = f.task(std::slice::from_ref(&state));
6110 task.resume_override = Some(crate::queue::OperatorResume {
6111 question_id: "fixture-pin".into(),
6112 at: Fx::now(),
6113 conductor_rehold: None,
6114 forced: true,
6115 pinned_run: Some(state.id.clone()),
6116 });
6117 crate::queue::Queue::at(f.home.join("queue"))
6118 .put(&mut task)
6119 .unwrap();
6120 }
6121 "task-replacement" => {
6122 let mut task = f.task(std::slice::from_ref(&state));
6123 task.replacement = Some(crate::merge_decision::Replacement {
6124 question: "fixture-replacement".into(),
6125 run: state.id.clone(),
6126 pr: "https://example.invalid/pull/1".into(),
6127 agent: "worker".into(),
6128 context: String::new(),
6129 });
6130 crate::queue::Queue::at(f.home.join("queue"))
6131 .put(&mut task)
6132 .unwrap();
6133 }
6134 "malformed-orig-head" => {
6135 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6136 .await
6137 .unwrap();
6138 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6139 .unwrap();
6140 }
6141 "grace" => disk.fold_grace_secs = 3600,
6142 "resumable" => state.status = RunStatus::Blocked,
6143 "archive-symlink" => {
6144 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6145 std::fs::create_dir_all(&artifacts).unwrap();
6146 let outside = f._temp.path().join("outside");
6147 std::fs::create_dir(&outside).unwrap();
6148 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6149 .unwrap();
6150 }
6151 "root-symlink" => {
6152 let root = state.worktree_root();
6153 let outside = f._temp.path().join("outside-root");
6154 std::fs::rename(&root, &outside).unwrap();
6155 std::os::unix::fs::symlink(&outside, &root).unwrap();
6156 }
6157 "checkpoint" => {
6158 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6159 .await
6160 .unwrap();
6161 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6162 }
6163 "symlink" => {
6164 std::fs::remove_file(review.join("source")).unwrap();
6165 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6166 }
6167 "retained" => std::fs::write(
6168 f.home
6169 .join("runs")
6170 .join(&state.id)
6171 .join("retain-recovery-sources"),
6172 "owner",
6173 )
6174 .unwrap(),
6175 "unreadable" => {
6176 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6177 }
6178 "unknown-field" | "unknown-task-field" => {}
6179 _ => unreachable!(),
6180 }
6181 state.save_under(&f.home).unwrap();
6182 if kind == "unknown-field" {
6183 let path = f.home.join("runs").join(&state.id).join("run.json");
6184 let mut raw: serde_json::Value =
6185 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6186 raw["future_source_owner"] = serde_json::json!({"path": review});
6187 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6188 }
6189 if kind == "unknown-task-field" {
6190 let task = f.task(std::slice::from_ref(&state));
6191 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6192 let mut raw: serde_json::Value =
6193 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6194 raw["future_source_owner"] = serde_json::json!({"path": review});
6195 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6196 }
6197 let mut report = Report::default();
6198 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6199 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6200 assert!(review.exists(), "{kind}: {report:?}");
6201 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6202 drop(output);
6203 drop(owner);
6204 drop(driver);
6205 }
6206 }
6207
6208 #[tokio::test]
6209 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6210 let f = Fx::new().await;
6211 let state = inactive_source(&f, "ec04", true).await;
6212 let review = reviewer(&f, &state, &f.base).await;
6213 let mut report = Report::default();
6214 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6215 assert_eq!(report.evicted_copies, 1, "{report:?}");
6216 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6217 let ledger = artifacts.join("eviction-review-1.json");
6218 let mut record: serde_json::Value =
6219 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6220 record["phase"] = "pending".into();
6221 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6222 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6223 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6224 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6225 let mut blocked = Report::default();
6226 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6227 assert!(
6228 blocked
6229 .retained
6230 .iter()
6231 .any(|r| r.contains("corrupt snapshot")),
6232 "{blocked:?}"
6233 );
6234 assert_eq!(
6235 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6236 ["phase"],
6237 "pending"
6238 );
6239 assert!(
6240 restore_source(&archive, &f._temp.path().join("bad-restore"))
6241 .await
6242 .is_err()
6243 );
6244 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6245 let mut retried = Report::default();
6246 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6247 assert_eq!(
6248 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6249 ["phase"],
6250 "evicted"
6251 );
6252 assert_eq!(retried.evicted_copies, 0);
6253 assert!(!review.exists());
6254 let pending = archive.with_extension("pending");
6256 std::fs::rename(&archive, &pending).unwrap();
6257 assert_eq!(
6258 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6259 .await
6260 .unwrap(),
6261 archive
6262 );
6263 assert!(!pending.exists());
6264 }
6265
6266 #[tokio::test]
6267 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6268 let f = Fx::new().await;
6269 let mut empty = inactive_source(&f, "ec05", false).await;
6270 historical_review(&mut empty, &f.base);
6271 empty.status = RunStatus::Superseded;
6272 empty.save_under(&f.home).unwrap();
6273 let source = inactive_source(&f, "ec06", true).await;
6274 let task = f.task(&[empty.clone(), source.clone()]);
6275 let before = serde_json::to_value(&empty).unwrap();
6276 let mut report = Report::default();
6277 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6278 assert_eq!(report.evicted_copies, 1, "{report:?}");
6279 assert!(!empty.candidates[0].worktree.exists());
6280 assert!(source.candidates[0].worktree.exists());
6281 assert!(
6282 git::branch_exists(&f.repo, &empty.candidates[0].branch)
6283 .await
6284 .unwrap()
6285 );
6286 assert_eq!(
6287 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6288 before
6289 );
6290 let mut retry = Report::default();
6291 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6292 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6293 assert!(!empty.worktree_root().exists());
6294 assert_eq!(
6295 crate::queue::Queue::at(f.home.join("queue"))
6296 .get(&task.id)
6297 .unwrap()
6298 .runs,
6299 task.runs
6300 );
6301 }
6302 #[tokio::test]
6303 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6304 fn interrupt(_: &Path) -> Result<()> {
6305 anyhow::bail!("fixture interruption before unlink")
6306 }
6307 fn vanish(path: &Path) -> Result<()> {
6308 std::fs::remove_dir_all(path)?;
6309 anyhow::bail!("fixture interruption after checkout unlink")
6310 }
6311 for stage in [
6312 "intent",
6313 "missing-registration",
6314 "changed-admin",
6315 "missing-archive",
6316 "wrong-identity",
6317 ] {
6318 let f = Fx::new().await;
6319 let state = inactive_source(&f, "ec07", true).await;
6320 let review = reviewer(&f, &state, &f.base).await;
6321 let admin = PathBuf::from(
6322 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6323 .await
6324 .unwrap(),
6325 );
6326 let mut interrupted = Report::default();
6327 copies::evict_with(
6328 &f.home,
6329 &Fx::disk(),
6330 Fx::now(),
6331 &mut interrupted,
6332 if stage == "missing-registration" || stage == "changed-admin" {
6333 vanish
6334 } else {
6335 interrupt
6336 },
6337 )
6338 .await;
6339 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6340 let ledger = f
6341 .home
6342 .join("runs")
6343 .join(&state.id)
6344 .join("artifacts/eviction-review-1.json");
6345 let record: serde_json::Value =
6346 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6347 assert_eq!(record["phase"], "pending");
6348 if stage == "changed-admin" {
6349 std::fs::write(admin.join("owner-note"), "keep").unwrap();
6350 }
6351 if stage == "missing-archive" {
6352 std::fs::remove_file(
6353 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6354 )
6355 .unwrap();
6356 }
6357 if stage == "wrong-identity" {
6358 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6359 .await
6360 .unwrap();
6361 let other =
6362 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6363 .await
6364 .unwrap();
6365 let original = Path::new(record["evidence"].as_str().unwrap());
6366 for name in [
6367 "manifest.json",
6368 "objects.pack",
6369 "change.patch",
6370 "change.index",
6371 ] {
6372 std::fs::copy(other.join(name), original.join(name)).unwrap();
6373 }
6374 }
6375 let mut retried = Report::default();
6376 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6377 let phase =
6378 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6379 .unwrap()["phase"]
6380 .clone();
6381 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6382 assert_eq!(phase, "pending", "{stage}: {retried:?}");
6383 assert!(!retried.retained.is_empty());
6384 if stage == "changed-admin" {
6385 assert!(admin.join("owner-note").exists());
6386 } else {
6387 assert!(review.exists());
6388 }
6389 } else {
6390 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6391 assert!(!review.exists());
6392 assert!(!admin.exists());
6393 }
6394 }
6395 }
6396 #[tokio::test]
6397 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
6398 let f = Fx::new().await;
6399 let mut runs = Vec::new();
6400 let mut paths = Vec::new();
6401 for tag in ["ec10", "ec11", "ec12", "ec13"] {
6402 let state = inactive_source(&f, tag, true).await;
6403 for name in ["judge-1", "advisor-1"] {
6404 let path = state.worktree_root().join(name);
6405 git::worktree_add_detached(&f.repo, &path, &f.base)
6406 .await
6407 .unwrap();
6408 paths.push(path);
6409 }
6410 runs.push(state);
6411 }
6412 let task = f.task(&runs);
6413 let mut report = Report::default();
6414 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6415 assert_eq!(report.evicted_copies, 5, "{report:?}");
6416 let mut next = Report::default();
6417 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6418 assert_eq!(next.evicted_copies, 3, "{next:?}");
6419 assert!(paths.iter().all(|p| !p.exists()));
6420 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
6421 assert_eq!(
6422 crate::queue::Queue::at(f.home.join("queue"))
6423 .get(&task.id)
6424 .unwrap()
6425 .runs,
6426 task.runs
6427 );
6428 let limited = Fx::new().await;
6429 let state = inactive_source(&limited, "ec15", true).await;
6430 let copy = reviewer(&limited, &state, &limited.base).await;
6431 let evidence = limited
6432 .home
6433 .join("runs")
6434 .join(&state.id)
6435 .join("artifacts/source-evidence");
6436 std::fs::create_dir_all(&evidence).unwrap();
6437 let retained = evidence.join("reserved-evidence");
6438 std::fs::File::create(&retained)
6439 .unwrap()
6440 .set_len(64 * 1024 * 1024)
6441 .unwrap();
6442 let mut blocked = Report::default();
6443 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6444 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
6445 assert!(copy.exists());
6446 assert_eq!(
6447 std::fs::metadata(&retained).unwrap().len(),
6448 64 * 1024 * 1024
6449 );
6450 assert!(
6451 blocked
6452 .retained
6453 .iter()
6454 .any(|r| r.contains("per-run storage limit")),
6455 "{blocked:?}"
6456 );
6457 }
6458 #[tokio::test]
6459 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
6460 let f = Fx::new().await;
6461 let mut state = inactive_source(&f, "ec14", true).await;
6462 historical_review(&mut state, &f.base);
6463 historical_review(&mut state, &f.base);
6464 state.status = RunStatus::Superseded;
6465 state.save_under(&f.home).unwrap();
6466 let review = reviewer(&f, &state, &f.base).await;
6467 let task = f.task(std::slice::from_ref(&state));
6468 let path = f.home.join("runs").join(&state.id).join("run.json");
6469 let mut raw: serde_json::Value =
6470 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6471 raw["schema"] = 10.into();
6472 raw.as_object_mut().unwrap().remove("output_activity_at");
6473 raw["resource_wait"] = serde_json::Value::Null;
6476 raw["resource_block"] = serde_json::Value::Null;
6477 raw["driver_claimed"] = serde_json::Value::Null;
6478 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6479 let graph = raw["config"]["graph"].as_object_mut().unwrap();
6481 let count = graph.remove("implementers").unwrap();
6482 graph.insert("candidates".into(), count);
6483 let queue = crate::queue::Queue::at(f.home.join("queue"));
6484 let task_path = queue.path_of(&task.id);
6485 let mut task_raw: serde_json::Value =
6486 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
6487 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6488 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
6489 std::fs::write(&task_path, &task_historical).unwrap();
6490 let historical = serde_json::to_vec_pretty(&raw).unwrap();
6491 std::fs::write(&path, &historical).unwrap();
6492 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6493 assert_eq!(report.evicted_copies, 1, "{report:?}");
6494 assert!(!review.exists());
6495 assert_eq!(std::fs::read(&path).unwrap(), historical);
6496 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
6497 assert!(state.candidates[0].worktree.exists());
6498 assert_eq!(
6499 crate::queue::Queue::at(f.home.join("queue"))
6500 .get(&task.id)
6501 .unwrap()
6502 .runs,
6503 task.runs
6504 );
6505 }
6506 #[tokio::test]
6507 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
6508 let f = Fx::new().await;
6509 let state = inactive_source(&f, "ed01", true).await;
6510 let review = reviewer(&f, &state, &f.base).await;
6511 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
6512 git::git(&review, &["add", "."]).await.unwrap();
6513 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6514 let tip = git::git(
6515 &review,
6516 &[
6517 "commit-tree",
6518 &tree,
6519 "-p",
6520 &f.base,
6521 "-m",
6522 "temporary detached source",
6523 ],
6524 )
6525 .await
6526 .unwrap();
6527 git::git(&review, &["checkout", "--detach", &tip])
6528 .await
6529 .unwrap();
6530 git::git(&review, &["checkout", "--detach", &f.base])
6531 .await
6532 .unwrap();
6533 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
6534 .await
6535 .unwrap();
6536 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
6537 .await
6538 .unwrap();
6539 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
6540 .await
6541 .unwrap();
6542 assert!(!shown.lines().any(|oid| oid == tip));
6543 let admin = PathBuf::from(
6544 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6545 .await
6546 .unwrap(),
6547 );
6548 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
6549 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
6550 fn interrupt(_: &Path) -> Result<()> {
6551 anyhow::bail!("fixture interruption before unlink")
6552 }
6553 let mut pending = Report::default();
6554 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
6555 assert!(review.exists());
6556 let ledger = f
6557 .home
6558 .join("runs")
6559 .join(&state.id)
6560 .join("artifacts/eviction-review-1.json");
6561 let complete = std::fs::read(&ledger).unwrap();
6562 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
6563 incomplete["history"]
6564 .as_object_mut()
6565 .unwrap()
6566 .remove(&tip)
6567 .unwrap();
6568 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
6569 let mut blocked = Report::default();
6570 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6571 assert!(review.exists());
6572 assert!(
6573 blocked
6574 .retained
6575 .iter()
6576 .any(|reason| reason.contains("incomplete reflog checkpoint")),
6577 "{blocked:?}"
6578 );
6579 std::fs::write(&ledger, complete).unwrap();
6580 let mut report = Report::default();
6581 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6582 assert_eq!(report.evicted_copies, 1, "{report:?}");
6583 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6584 .await
6585 .unwrap();
6586 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6587 assert!(
6588 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
6589 .await
6590 .unwrap()
6591 .ok()
6592 );
6593 std::fs::remove_dir_all(&f.repo).unwrap();
6594 let destination = f._temp.path().join("restore-old-only");
6595 restore_seat(&f.home, &state.id, &tip, &destination)
6596 .await
6597 .unwrap();
6598 assert_eq!(
6599 std::fs::read(destination.join("old-only")).unwrap(),
6600 b"old-only\0binary\xff"
6601 );
6602 }
6603
6604 #[tokio::test]
6605 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
6606 fn interrupt(_: &Path) -> Result<()> {
6607 anyhow::bail!("fixture interruption before unlink")
6608 }
6609 fn vanish(path: &Path) -> Result<()> {
6610 std::fs::remove_dir_all(path)?;
6611 anyhow::bail!("fixture interruption before unregistration")
6612 }
6613 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
6614 let f = Fx::new().await;
6615 let mut state = inactive_source(&f, "ed02", true).await;
6616 state.driver_exited = false;
6617 state.driver_claimed = Some(true);
6618 state.driver_pid = Some(std::process::id());
6619 state.driver_started_at = crate::proc::process_started_at(std::process::id());
6620 state.begin_pass();
6621 state.status = RunStatus::Superseded;
6622 state.save_under(&f.home).unwrap();
6623 let review = reviewer(&f, &state, &f.base).await;
6624 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
6627 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
6628 std::fs::write(
6629 f.home.join("runs").join(&state.id).join("driver.json"),
6630 serde_json::to_vec(&identity).unwrap(),
6631 )
6632 .unwrap();
6633 let live = if stage == "live-driver" {
6634 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
6635 } else {
6636 None
6637 };
6638 assert_eq!(
6639 state.liveness_under(&f.home, false),
6640 if live.is_some() {
6641 Liveness::Live
6642 } else {
6643 Liveness::Dead
6644 }
6645 );
6646 let mut report = Report::default();
6647 if stage == "intent" || stage == "missing-registration" {
6648 copies::evict_with(
6649 &f.home,
6650 &Fx::disk(),
6651 Fx::now(),
6652 &mut report,
6653 if stage == "intent" { interrupt } else { vanish },
6654 )
6655 .await;
6656 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
6657 let mut retry = Report::default();
6658 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6659 assert!(!review.exists(), "{stage}: {retry:?}");
6660 let ledger = f
6661 .home
6662 .join("runs")
6663 .join(&state.id)
6664 .join("artifacts/eviction-review-1.json");
6665 let receipt: serde_json::Value =
6666 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
6667 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
6668 } else {
6669 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6670 assert_eq!(
6671 report.evicted_copies,
6672 usize::from(live.is_none()),
6673 "{stage}: {report:?}"
6674 );
6675 assert_eq!(review.exists(), live.is_some());
6676 }
6677 drop(live);
6678 assert!(
6679 !RunState::load_under(&state.id, &f.home)
6680 .unwrap()
6681 .driver_exited
6682 );
6683 }
6684 }
6685}