Skip to main content

magi/
retention.rs

1//! Bounded reclamation of retained attempts and Cargo outputs. Source recovery
2//! and build eviction have separate authority: evicting a target never retires a run.
3mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7    let artifacts = home.join("runs").join(&state.id).join("artifacts");
8    if !artifacts.try_exists()? {
9        return Ok(false);
10    }
11    ensure!(
12        plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13        "noncanonical artifacts"
14    );
15    for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16        ensure!(i < 4096, "eviction record inspection limit exceeded");
17        let name = entry?.file_name();
18        let name = name.to_string_lossy();
19        if name.starts_with("eviction-") && name.ends_with(".json") {
20            return Ok(true);
21        }
22    }
23    Ok(false)
24}
25
26/// Re-materialize evicted read seats before execution can consume their paths.
27/// Caller owns the run's driver/owner leases and repository coordination.
28pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29    copies::restore_execution_seats(home, state).await
30}
31
32/// Read the binary diff after verifying its independent source archive.
33pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34    crate::fold_evidence::validate_snapshot(evidence).await?;
35    Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38/// Restore immutable checkout evidence into a new inspection repository.
39pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40    crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43/// List archived checkout tips for a run, validating their durable evidence.
44pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
45    copies::seat_evidence(home, id).await
46}
47
48/// Validated archived heads and seat labels for the run's evidence browser.
49pub async fn seat_heads(
50    home: &Path,
51    id: &str,
52) -> Result<std::collections::BTreeMap<String, String>> {
53    copies::seat_heads(home, id).await
54}
55
56/// Read a recorded seat's diff without relying on its checkout or Git objects.
57pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
58    copies::seat_diff(home, id, head).await
59}
60
61/// Restore a recorded checkout tip without changing run or recovery authority.
62pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
63    copies::restore_seat(home, id, head, destination).await
64}
65
66use crate::{
67    git,
68    proc::Quiet as _,
69    run::{Liveness, RunState},
70};
71use anyhow::{Context, Result, ensure};
72use jiff::Timestamp;
73use serde::{Deserialize, Serialize};
74use std::{
75    fs::{File, OpenOptions},
76    path::{Path, PathBuf},
77};
78
79/// An OS lease on a worktree's outputs, independent of repository coordination.
80/// Kept by the process cleanup owner until descendants terminate, including
81/// reparented tools retaining the inherited liveness witness.
82#[derive(Debug)]
83pub(crate) struct OutputLease {
84    _file: std::sync::Arc<OutputFile>,
85    _activity: bool,
86}
87
88#[derive(Debug)]
89struct OutputFile(File);
90
91impl std::ops::Deref for OutputFile {
92    type Target = File;
93    fn deref(&self) -> &File {
94        &self.0
95    }
96}
97
98#[cfg(unix)]
99impl Drop for OutputFile {
100    fn drop(&mut self) {
101        use std::os::fd::AsRawFd;
102        // SAFETY: the final shared owner releases this live descriptor's lock.
103        unsafe {
104            libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
105        }
106    }
107}
108
109impl OutputLease {
110    #[cfg(unix)]
111    pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
112        use std::os::fd::AsRawFd;
113        self._file.as_raw_fd()
114    }
115
116    /// A separate exclusive lock prevents nested/concurrent pre-clean operations.
117    pub(crate) fn pre_clean(dir: &Path) -> Result<Self> {
118        let file = OpenOptions::new()
119            .create(true)
120            .truncate(false)
121            .read(true)
122            .write(true)
123            .open(dir.join("magi-pre-clean.lock"))?;
124        crate::pr_head::try_lock_file(&file)
125            .context("pre-clean already running; retry after descendants terminate")?;
126        Ok(Self {
127            _file: std::sync::Arc::new(OutputFile(file)),
128            _activity: false,
129        })
130    }
131
132    pub(crate) fn duplicate(&self) -> Self {
133        Self {
134            _file: std::sync::Arc::clone(&self._file),
135            _activity: false,
136        }
137    }
138
139    pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
140        if !worktree.join(".git").is_file() {
141            return Ok(None);
142        }
143        Self::take(worktree, false).map(Some)
144    }
145    pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
146        use std::hash::{Hash, Hasher};
147        let mut hash = std::collections::hash_map::DefaultHasher::new();
148        let mut normalized = cache.to_owned();
149        let mut missing = Vec::new();
150        while !normalized.exists() {
151            let Some(name) = normalized.file_name() else {
152                break;
153            };
154            missing.push(name.to_owned());
155            if !normalized.pop() {
156                break;
157            }
158        }
159        normalized = normalized.canonicalize().unwrap_or(normalized);
160        for part in missing.into_iter().rev() {
161            normalized.push(part);
162        }
163        normalized.hash(&mut hash);
164        let dir = home.join("cache-leases");
165        std::fs::create_dir_all(&dir)?;
166        Self::lock(
167            &dir.join(format!("outputs-{:x}.lock", hash.finish())),
168            exclusive,
169        )
170    }
171    /// Reserve headroom once per volume while admitting a managed writer.
172    /// These are estimates, not filesystem quotas. The ordinary emergency
173    /// monitor remains necessary for writes exceeding the reservation.
174    #[cfg(unix)]
175    pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
176        Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
177    }
178
179    #[cfg(unix)]
180    pub(crate) fn admit_with(
181        home: &Path,
182        paths: &[PathBuf],
183        min_free: u64,
184        free_bytes: impl Fn(&Path) -> Result<u64>,
185    ) -> Result<Vec<Self>> {
186        use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
187        if min_free == 0 {
188            return Ok(Vec::new());
189        }
190        const RESERVE: u64 = 1024 * 1024 * 1024;
191        std::fs::create_dir_all(home)?;
192        let dir = home.canonicalize()?.join("disk-admission");
193        std::fs::create_dir_all(&dir)?;
194        ensure!(
195            plain_dir(&dir) && dir.canonicalize()? == dir,
196            "disk admission directory is not canonical"
197        );
198        let mut devices = std::collections::BTreeSet::new();
199        let mut reservations = Vec::new();
200        for path in paths {
201            let probe = path
202                .ancestors()
203                .find(|p| p.exists())
204                .context("disk admission path missing")?;
205            let device = std::fs::metadata(probe)?.dev();
206            if !devices.insert(device) {
207                continue;
208            }
209            let open = |path: &Path| -> Result<File> {
210                Ok(OpenOptions::new()
211                    .read(true)
212                    .write(true)
213                    .create(true)
214                    .truncate(false)
215                    .custom_flags(libc::O_NOFOLLOW)
216                    .open(path)?)
217            };
218            // Explicitly unlock on return. A concurrent fork can briefly
219            // inherit this descriptor before exec closes it; closing only the
220            // parent's copy would leave the namespace spuriously busy.
221            let admission = Self {
222                _file: std::sync::Arc::new(OutputFile(open(&dir.join(format!("{device}.lock")))?)),
223                _activity: false,
224            };
225            let started = std::time::Instant::now();
226            loop {
227                match crate::pr_head::try_lock_file(&admission._file) {
228                    Ok(()) => break,
229                    Err(error)
230                        if matches!(
231                            error.kind(),
232                            std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
233                        ) && started.elapsed() < std::time::Duration::from_secs(1) =>
234                    {
235                        std::thread::sleep(std::time::Duration::from_millis(5));
236                    }
237                    Err(error) => {
238                        return Err(error)
239                            .context("disk admission busy or unavailable; retry without dispatch");
240                    }
241                }
242            }
243            let mut active = 0u64;
244            let mut available = None;
245            for slot in 0..1024 {
246                let file = dir.join(format!("{device}-{slot}.reserve"));
247                if !file.exists() {
248                    if available.is_none() {
249                        available = Some(file);
250                    }
251                    continue;
252                }
253                let file = open(&file)?;
254                match crate::pr_head::try_lock_file(&file) {
255                    Ok(()) if available.is_none() => {
256                        available = Some(dir.join(format!("{device}-{slot}.reserve")))
257                    }
258                    Ok(()) => {}
259                    Err(_) => active = active.saturating_add(RESERVE),
260                }
261            }
262            let free = free_bytes(probe)?;
263            let required = min_free.saturating_add(active).saturating_add(RESERVE);
264            ensure!(
265                free >= required,
266                "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
267            );
268            let file = open(&available.context("disk admission reservation slots exhausted")?)?;
269            crate::pr_head::try_lock_file(&file)?;
270            reservations.push(Self {
271                _file: std::sync::Arc::new(OutputFile(file)),
272                _activity: false,
273            });
274        }
275        Ok(reservations)
276    }
277
278    #[cfg(not(unix))]
279    pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
280        // Registered seats already fail closed without Unix containment.
281        Ok(Vec::new())
282    }
283
284    pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
285        let out = std::process::Command::new("git")
286            .args(["rev-parse", "--absolute-git-dir"])
287            .current_dir(worktree)
288            .quiet()
289            .output()?;
290        ensure!(
291            out.status.success(),
292            "cannot identify output lease directory"
293        );
294        let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
295        Self::lock(&dir.join("magi-output.lock"), exclusive)
296    }
297    fn lock(path: &Path, exclusive: bool) -> Result<Self> {
298        let file = OpenOptions::new()
299            .create(true)
300            .truncate(false)
301            .read(true)
302            .write(true)
303            .open(path)
304            .with_context(|| format!("output lease: open {}", path.display()))?;
305        #[cfg(unix)]
306        {
307            use std::os::fd::AsRawFd;
308            let mode = if exclusive {
309                libc::LOCK_EX
310            } else {
311                libc::LOCK_SH
312            };
313            // SAFETY: live file descriptor, no retained pointers.
314            ensure!(
315                unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
316                "output lease: outputs are in use or being reclaimed: {}",
317                path.display()
318            );
319        }
320        #[cfg(not(unix))]
321        let _ = exclusive;
322        if !exclusive {
323            // Empty legacy/maintenance-only leases carry no activity witness.
324            file.set_len(1)?;
325            file.set_modified(std::time::SystemTime::now())
326                .with_context(|| format!("record output activity {}", path.display()))?;
327        }
328        Ok(Self {
329            _file: std::sync::Arc::new(OutputFile(file)),
330            _activity: !exclusive,
331        })
332    }
333}
334#[cfg(unix)]
335impl Drop for OutputLease {
336    fn drop(&mut self) {
337        if self._activity
338            && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
339        {
340            // If release time cannot be recorded, retain a nonempty invalid
341            // witness rather than trusting the old acquisition timestamp.
342            let _ = self._file.set_len(2);
343            tracing::warn!("could not record output lease release activity: {error}");
344        }
345    }
346}
347
348#[derive(Default, Debug, Serialize, Deserialize)]
349#[serde(default)]
350pub(crate) struct Report {
351    pub checked_at: Option<Timestamp>,
352    pub limit_bytes: u64,
353    pub reclaim_budget_bytes: u64,
354    /// Logical inventory, including protected and unproven data.
355    pub measured_bytes: u64,
356    /// Logical bytes unlinked, not a measurement of physical disk recovery.
357    pub freed_bytes: u64,
358    pub measured_allocated_file_bytes: Option<u64>,
359    /// Candidates proven under locks in this pass; uninspected data is excluded.
360    pub proven_reclaimable_bytes: u64,
361    /// Estimated allocated blocks recoverable if every proven link is removed.
362    /// Inodes with links outside the candidate set contribute zero.
363    pub proven_reclaimable_allocated_bytes: Option<u64>,
364    pub protected_or_unproven_bytes: u64,
365    /// Deduplicated allocated file blocks whose final link was unlinked. Open
366    /// descriptors and shared extents can prevent an equivalent physical gain.
367    pub estimated_released_allocated_bytes: Option<u64>,
368    /// Observed free-space change on the home volume; concurrent writes, clones
369    /// and deferred reclamation mean this is not attributable to this sweep.
370    pub observed_home_free_change_bytes: Option<i64>,
371    pub retired: usize,
372    pub evicted_copies: usize,
373    pub removed_empty_roots: usize,
374    pub source_unlinked_logical_bytes: u64,
375    pub source_unlinked_allocated_bytes: Option<u64>,
376    pub source_evidence_added_logical_bytes: u64,
377    pub source_evidence_added_allocated_bytes: Option<u64>,
378    pub retained: Vec<String>,
379    pub failures: Vec<String>,
380}
381
382#[cfg(unix)]
383fn allocated_inventory(
384    root: &Path,
385    seen: &mut std::collections::BTreeSet<(u64, u64)>,
386) -> Result<u64> {
387    use std::os::unix::fs::MetadataExt;
388    ensure!(
389        plain_dir(root) && root.canonicalize()? == root,
390        "allocated inventory root is not canonical"
391    );
392    let mut dirs = vec![root.to_owned()];
393    let mut bytes = 0u64;
394    while let Some(dir) = dirs.pop() {
395        for entry in std::fs::read_dir(dir)? {
396            let path = entry?.path();
397            let metadata = std::fs::symlink_metadata(&path)?;
398            if metadata.is_dir() {
399                dirs.push(path);
400            } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
401                bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
402            }
403        }
404    }
405    Ok(bytes)
406}
407
408#[derive(Default)]
409struct ReleasedAllocation {
410    #[cfg(unix)]
411    files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
412    #[cfg(unix)]
413    removed: std::collections::BTreeMap<(u64, u64), u64>,
414}
415
416impl ReleasedAllocation {
417    fn of(paths: &[PathBuf]) -> Result<Self> {
418        #[cfg(unix)]
419        {
420            use std::os::unix::fs::MetadataExt;
421            let mut files = std::collections::BTreeMap::new();
422            for path in paths {
423                let m = std::fs::symlink_metadata(path)?;
424                ensure!(m.is_file(), "allocation candidate is not regular");
425                files.insert(
426                    path.clone(),
427                    (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
428                );
429            }
430            Ok(Self {
431                files,
432                removed: Default::default(),
433            })
434        }
435        #[cfg(not(unix))]
436        {
437            let _ = paths;
438            Ok(Self::default())
439        }
440    }
441    fn reclaimable_allocated_bytes(&self) -> Option<u64> {
442        #[cfg(unix)]
443        {
444            let mut inodes = std::collections::BTreeMap::new();
445            for &(device, inode, links, blocks) in self.files.values() {
446                let entry = inodes
447                    .entry((device, inode))
448                    .or_insert((0u64, links, blocks));
449                entry.0 += 1;
450            }
451            Some(
452                inodes
453                    .values()
454                    .filter(|(count, links, _)| count == links)
455                    .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
456            )
457        }
458        #[cfg(not(unix))]
459        {
460            None
461        }
462    }
463
464    fn unlinked(&mut self, path: &Path) -> Option<u64> {
465        #[cfg(unix)]
466        {
467            let (device, inode, links, blocks) = self.files[path];
468            let removed = self.removed.entry((device, inode)).or_default();
469            *removed += 1;
470            Some(if *removed == links { blocks } else { 0 })
471        }
472        #[cfg(not(unix))]
473        {
474            let _ = path;
475            None
476        }
477    }
478}
479
480fn plain_dir(path: &Path) -> bool {
481    std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
482}
483
484/// Old run records outlive their checkout and output directories. Only an
485/// absent path means nothing to reclaim; inaccessible paths remain errors.
486fn run_paths_missing(state: &RunState) -> Result<bool> {
487    for path in [state.worktree_root(), state.repo.clone()] {
488        match std::fs::metadata(&path) {
489            Ok(_) => {}
490            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
491            Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
492        }
493    }
494    Ok(false)
495}
496
497/// Enumerate actual registered worktrees, not just candidates in run.json.
498async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
499    let root = state.worktree_root();
500    if !plain_dir(&root) {
501        return Ok(Vec::new());
502    }
503    let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
504    registered_worktrees(state, &list)
505}
506
507fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
508    let root = state.worktree_root();
509    if !plain_dir(&root) {
510        return Ok(Vec::new());
511    }
512    let root = root.canonicalize()?;
513    let mut paths = Vec::new();
514    for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
515        let path = PathBuf::from(path);
516        #[cfg(windows)]
517        let path = if plain_dir(&path) {
518            git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
519        } else {
520            continue;
521        };
522        if plain_dir(&path)
523            && path.parent() == Some(root.as_path())
524            && path.canonicalize()? == path
525            && path.join(".git").is_file()
526        {
527            paths.push(path);
528        }
529    }
530    Ok(paths)
531}
532
533/// Recognize Cargo's generated directory, never merely an ignored directory.
534/// Unknown root entries are retained; cache identity/profile paths fail closed.
535async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
536    validate_cargo_target(worktree, false).await
537}
538
539async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
540    let target = worktree.join("target");
541    ensure!(plain_dir(&target), "target absent or not a plain directory");
542    ensure!(target.canonicalize()? == target, "target path changed");
543    ensure!(
544        git::git(worktree, &["ls-files", "--", "target"])
545            .await?
546            .is_empty(),
547        "tracked target contents"
548    );
549    ensure!(
550        git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
551            .await?
552            .ok(),
553        "target is not ignored"
554    );
555    cache_tag(&target, allow_missing_tag)?;
556    let path = target.join(".rustc_info.json");
557    plain_file(&path, "rustc identity")?;
558    let info: serde_json::Value = serde_json::from_slice(
559        &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
560    )
561    .with_context(|| format!("parse rustc identity {}", path.display()))?;
562    ensure!(
563        info.get("rustc_fingerprint").is_some(),
564        "unknown Cargo identity {}",
565        path.display()
566    );
567    if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
568        ensure!(
569            info.get("rustc_fingerprint")
570                .is_some_and(serde_json::Value::is_u64)
571                && info
572                    .get("outputs")
573                    .and_then(serde_json::Value::as_object)
574                    .is_some_and(|outputs| outputs.values().any(|output| {
575                        output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
576                            && output
577                                .get("stdout")
578                                .and_then(serde_json::Value::as_str)
579                                .is_some_and(|stdout| stdout.starts_with("rustc "))
580                            && output
581                                .get("stderr")
582                                .is_some_and(serde_json::Value::is_string)
583                    }))
584                && info
585                    .get("successes")
586                    .is_some_and(serde_json::Value::is_object),
587            "missing-tag rustc identity structure invalid: {}",
588            path.display()
589        );
590        let mut profiles = 0;
591        for profile in ["debug", "release"] {
592            let dir = target.join(profile);
593            if !dir.exists() {
594                continue;
595            }
596            for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
597                ensure!(
598                    plain_dir(&subdir) && subdir.canonicalize()? == subdir,
599                    "missing-tag profile proof: {} is not a plain directory",
600                    subdir.display()
601                );
602            }
603            plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
604            profiles += 1;
605        }
606        ensure!(
607            profiles > 0,
608            "missing-tag proof: no Cargo profile in {}",
609            target.display()
610        );
611    }
612    for name in ["debug", "release"] {
613        let path = target.join(name);
614        match std::fs::symlink_metadata(&path) {
615            Ok(metadata) => ensure!(
616                metadata.is_dir() && path.canonicalize()? == path,
617                "profile validation: not a plain directory: {}",
618                path.display()
619            ),
620            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
621            Err(error) => {
622                return Err(error)
623                    .with_context(|| format!("profile validation: inspect {}", path.display()));
624            }
625        }
626    }
627    Ok(target)
628}
629
630fn plain_file(path: &Path, stage: &str) -> Result<()> {
631    let meta = std::fs::symlink_metadata(path)
632        .with_context(|| format!("{stage}: inspect {}", path.display()))?;
633    ensure!(
634        meta.is_file() && path.canonicalize()? == path,
635        "{stage}: not a plain file: {}",
636        path.display()
637    );
638    Ok(())
639}
640
641fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
642    let path = target.join("CACHEDIR.TAG");
643    match std::fs::symlink_metadata(&path) {
644        Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
645            return Ok(());
646        }
647        result => {
648            result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
649        }
650    }
651    plain_file(&path, "cache tag")?;
652    let tag = std::fs::read_to_string(&path)
653        .with_context(|| format!("cache tag: read {}", path.display()))?;
654    ensure!(
655        tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
656        "unknown Cargo cache tag: {}",
657        path.display()
658    );
659    Ok(())
660}
661
662/// Observe compiler metadata and artifacts under the output and Cargo locks.
663/// Unknown contents never become deletion authority. Unreadable timestamps or
664/// a future timestamp retain outputs, even when the configured grace is zero.
665fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
666    let mut dirs = vec![target.to_owned()];
667    let mut entries = 0;
668    while let Some(dir) = dirs.pop() {
669        for entry in std::fs::read_dir(&dir)
670            .with_context(|| format!("output activity: list {}", dir.display()))?
671        {
672            let path = entry?.path();
673            entries += 1;
674            ensure!(
675                entries <= 100_000,
676                "output activity inventory too large: {}",
677                target.display()
678            );
679            let metadata = std::fs::symlink_metadata(&path)
680                .with_context(|| format!("output activity: inspect {}", path.display()))?;
681            if metadata.is_dir() {
682                // Only supported Cargo profiles supply execution evidence.
683                if dir != target
684                    || matches!(
685                        path.file_name().and_then(|s| s.to_str()),
686                        Some("debug" | "release")
687                    )
688                {
689                    dirs.push(path);
690                }
691            } else if metadata.is_file()
692                && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
693            {
694                let at =
695                    Timestamp::try_from(metadata.modified().with_context(|| {
696                        format!("output activity: modified {}", path.display())
697                    })?)?;
698                ensure!(
699                    crate::clean::due(now, at, grace),
700                    "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
701                    path.display()
702                );
703            }
704        }
705    }
706    Ok(())
707}
708
709pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
710    use std::io::Read as _;
711    let mut header = [0; 8];
712    let n = File::open(path)?.read(&mut header)?;
713    let header = &header[..n];
714    Ok(header.starts_with(b"\x7fELF")
715        || header.starts_with(b"!<arch>\n")
716        || header.starts_with(b"rust\0\0\0")
717        || header.starts_with(b"MZ\x90\0")
718        || [
719            [0xfe, 0xed, 0xfa, 0xce],
720            [0xce, 0xfa, 0xed, 0xfe],
721            [0xfe, 0xed, 0xfa, 0xcf],
722            [0xcf, 0xfa, 0xed, 0xfe],
723            [0xca, 0xfe, 0xba, 0xbe],
724            [0xca, 0xfe, 0xba, 0xbf],
725        ]
726        .iter()
727        .any(|magic| header.starts_with(magic)))
728}
729
730/// Legacy tags identify a cache, not ownership of every file inside it.
731/// Only direct rustc outputs named by dep-info may be reclaimed. Build-script
732/// output, incremental caches, unknown files and the manifests themselves stay.
733fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
734    let mut outputs = std::collections::BTreeSet::new();
735    for profile in ["debug", "release"] {
736        for dir in [target.join(profile), target.join(profile).join("deps")] {
737            if !plain_dir(&dir) || dir.canonicalize()? != dir {
738                continue;
739            }
740            for entry in std::fs::read_dir(&dir)? {
741                let path = entry?.path();
742                if path.extension().is_none_or(|ext| ext != "d")
743                    || !std::fs::symlink_metadata(&path)?.is_file()
744                    || std::fs::metadata(&path)?.len() > 1024 * 1024
745                    || path.canonicalize()? != path
746                {
747                    continue;
748                }
749                let body = std::fs::read_to_string(&path)?;
750                for line in body.lines() {
751                    let Some((names, dependencies)) = line.split_once(": ") else {
752                        continue;
753                    };
754                    if dependencies.trim().is_empty() {
755                        continue;
756                    }
757                    // Rustc escapes spaces and backslashes in Make paths.
758                    // Unsupported Make syntax fails closed for that record.
759                    let mut words = Vec::new();
760                    let mut word = String::new();
761                    let mut escaped = false;
762                    for c in names.chars() {
763                        if escaped {
764                            word.push(c);
765                            escaped = false;
766                        } else if c == '\\' {
767                            escaped = true;
768                        } else if c.is_whitespace() {
769                            if !word.is_empty() {
770                                words.push(std::mem::take(&mut word));
771                            }
772                        } else {
773                            word.push(c);
774                        }
775                    }
776                    if escaped || word.contains(['$', '%']) {
777                        continue;
778                    }
779                    if !word.is_empty() {
780                        words.push(word);
781                    }
782                    for name in words {
783                        let output = PathBuf::from(name);
784                        let stem = path.file_stem().unwrap_or_default().to_string_lossy();
785                        let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
786                        let artifact_name =
787                            output_stem == stem || output_stem == format!("lib{stem}");
788                        let artifact_kind = output.extension().is_none_or(|ext| {
789                            matches!(
790                                ext.to_str(),
791                                Some(
792                                    "rlib"
793                                        | "rmeta"
794                                        | "so"
795                                        | "dylib"
796                                        | "a"
797                                        | "lib"
798                                        | "dll"
799                                        | "exe"
800                                        | "pdb"
801                                )
802                            )
803                        });
804                        if artifact_name
805                            && artifact_kind
806                            && output.parent() == Some(dir.as_path())
807                            && output.extension().is_none_or(|ext| ext != "d")
808                            && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
809                            && output.canonicalize().is_ok_and(|p| p == output)
810                            && artifact_signature(&output)?
811                        {
812                            outputs.insert(output);
813                        }
814                    }
815                }
816            }
817        }
818    }
819    Ok(outputs.into_iter().collect())
820}
821
822#[derive(Serialize, Deserialize)]
823struct CargoResidue {
824    target: PathBuf,
825    files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
826}
827
828// Keep the exact metadata as historical evidence. An old proof cannot adopt
829// newly added files, changed metadata, symlinks or a different worktree.
830fn residue_files(
831    target: &Path,
832    outputs: &[PathBuf],
833) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
834    let mut files = std::collections::BTreeMap::new();
835    let mut dirs = vec![target.to_owned()];
836    let mut bytes = 0usize;
837    let mut entries = 0usize;
838    while let Some(dir) = dirs.pop() {
839        for entry in std::fs::read_dir(dir)? {
840            let path = entry?.path();
841            entries += 1;
842            ensure!(entries <= 4096, "residue inventory too large");
843            let metadata = std::fs::symlink_metadata(&path)?;
844            ensure!(
845                !metadata.file_type().is_symlink(),
846                "symlink in Cargo residue"
847            );
848            if metadata.is_dir() {
849                dirs.push(path);
850            } else {
851                ensure!(metadata.is_file(), "non-file in Cargo residue");
852                if outputs.contains(&path) {
853                    continue;
854                }
855                ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
856                let body = std::fs::read(&path)?;
857                bytes = bytes.saturating_add(body.len());
858                ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
859                files.insert(path.strip_prefix(target)?.to_owned(), body);
860            }
861        }
862    }
863    Ok(files)
864}
865
866fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
867    Ok(home
868        .join("runs")
869        .join(id)
870        .join("build-residue")
871        .join(worktree.file_name().context("missing worktree name")?))
872}
873
874// Artifact eviction tolerates unrelated root data. Whole-target removal must
875// still reject it, including empty directories absent from the file inventory.
876fn residue_root(target: &Path) -> Result<()> {
877    for entry in std::fs::read_dir(target)? {
878        let name = entry?.file_name();
879        ensure!(
880            matches!(
881                name.to_str(),
882                Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
883            ),
884            "unrecognized Cargo residue root entry"
885        );
886    }
887    Ok(())
888}
889
890fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
891    cache_tag(target, false)?;
892    residue_root(target)?;
893    let files = residue_files(target, outputs)?;
894    for (relative, body) in &files {
895        let path = target.join(relative);
896        let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
897        let lock = matches!(
898            relative.to_str(),
899            Some("debug/.cargo-lock" | "release/.cargo-lock")
900        ) && body.is_empty();
901        let dep_info = path.extension().is_some_and(|ext| ext == "d")
902            && outputs.iter().any(|output| {
903                output.parent() == path.parent()
904                    && (output.file_stem() == path.file_stem()
905                        || output.file_stem().is_some_and(|stem| {
906                            stem.to_string_lossy()
907                                == format!(
908                                    "lib{}",
909                                    path.file_stem().unwrap_or_default().to_string_lossy()
910                                )
911                        }))
912            });
913        ensure!(identity || lock || dep_info, "unproven Cargo residue");
914    }
915    let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
916    std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
917    let tmp = proof.with_extension("tmp");
918    std::fs::write(
919        &tmp,
920        serde_json::to_vec(&CargoResidue {
921            target: target.to_owned(),
922            files,
923        })?,
924    )?;
925    std::fs::rename(tmp, proof)?;
926    Ok(())
927}
928
929pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
930    let target = cargo_target(worktree).await?;
931    residue_root(&target)?;
932    let locks = cargo_locks(&target)?;
933    let proof: CargoResidue =
934        serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
935    ensure!(
936        proof.target == target && proof.files == residue_files(&target, &[])?,
937        "Cargo residue changed or contains unproven files"
938    );
939    Ok(locks)
940}
941
942/// Supplemental exclusion for Cargo builds launched outside the managed seats.
943pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
944    let mut held = Vec::new();
945    for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
946        [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
947    }) {
948        let lock = target.join(profile).join(name);
949        match std::fs::symlink_metadata(&lock) {
950            Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
951            result => {
952                result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
953            }
954        }
955        {
956            plain_file(&lock, "Cargo lock")?;
957            let file = OpenOptions::new()
958                .read(true)
959                .write(true)
960                .open(&lock)
961                .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
962            crate::pr_head::try_lock_file(&file)
963                .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
964            held.push(file);
965        }
966    }
967    Ok(held)
968}
969
970/// All repositories share one cap and one per-pass deletion budget. The report
971/// survives a restart and explains every skipped run/target or failed operation.
972pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
973    let home = home.to_owned();
974    let disk = disk.clone();
975    let runtime = tokio::runtime::Handle::current();
976    tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
977        .await
978        .unwrap_or_else(|e| Report {
979            failures: vec![format!("retention worker failed: {e}")],
980            ..Report::default()
981        })
982}
983
984#[derive(Default, Serialize, Deserialize)]
985struct Measurement {
986    at: Option<Timestamp>,
987    bytes: u64,
988}
989
990async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
991    let mut report = Report {
992        checked_at: Some(now),
993        limit_bytes: disk.worktree_limit_bytes,
994        reclaim_budget_bytes: disk.worktree_reclaim_bytes,
995        ..Report::default()
996    };
997    if !cfg!(unix) {
998        report
999            .failures
1000            .push("Automatic output reclamation requires Unix process containment".into());
1001        return report;
1002    }
1003    let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
1004        return report;
1005    };
1006    let mut targets = Vec::new();
1007    let mut under_pressure = false;
1008    let mut registrations = std::collections::HashMap::new();
1009    let measurements_path = home.join("output-measurements.json");
1010    let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
1011        std::fs::read(&measurements_path)
1012            .ok()
1013            .and_then(|b| serde_json::from_slice(&b).ok())
1014            .unwrap_or_default();
1015    let mut observed = std::collections::BTreeSet::new();
1016    let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
1017        .into_iter()
1018        .flatten()
1019        .flatten()
1020        .map(|e| e.file_name().to_string_lossy().into_owned())
1021        .collect();
1022    ids.sort();
1023    for id in ids {
1024        if !crate::run::is_run_id(&id) {
1025            continue;
1026        }
1027        let state = match RunState::load_under(&id, home) {
1028            Ok(state) => state,
1029            Err(e) => {
1030                report
1031                    .retained
1032                    .push(format!("{id}: unreadable run ownership: {e:#}"));
1033                continue;
1034            }
1035        };
1036        if disk.worktree_limit_bytes == 0 {
1037            continue;
1038        }
1039        match run_paths_missing(&state) {
1040            Ok(true) => continue,
1041            Ok(false) => {}
1042            Err(e) => {
1043                report.failures.push(format!("{id}: {e:#}"));
1044                continue;
1045            }
1046        }
1047        let list = match registrations.entry(state.repo.clone()) {
1048            std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1049            std::collections::hash_map::Entry::Vacant(entry) => {
1050                match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1051                    Ok(list) => entry.insert(list),
1052                    Err(e) => {
1053                        // The checkout may disappear after the inventory check.
1054                        // Do not confuse a missing Git executable with that case.
1055                        if matches!(run_paths_missing(&state), Ok(true)) {
1056                            continue;
1057                        }
1058                        report.failures.push(format!("{id}: {e:#}"));
1059                        continue;
1060                    }
1061                }
1062            }
1063        };
1064        let paths = match registered_worktrees(&state, list) {
1065            Ok(paths) => paths,
1066            Err(e) => {
1067                report.failures.push(format!("{id}: {e:#}"));
1068                continue;
1069            }
1070        };
1071        for path in paths {
1072            if !path.join("target").exists() {
1073                continue;
1074            }
1075            let target = path.join("target");
1076            observed.insert(target.clone());
1077            let pressure =
1078                crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1079            under_pressure |= pressure;
1080            let measurement = measurements.entry(target).or_default();
1081            if pressure
1082                || measurement
1083                    .at
1084                    .is_none_or(|at| crate::clean::due(now, at, 300))
1085            {
1086                measurement.bytes = crate::disk::dir_size(&path.join("target"));
1087                measurement.at = Some(now);
1088            }
1089            let size = measurement.bytes;
1090            report.measured_bytes = report.measured_bytes.saturating_add(size);
1091            targets.push((state.output_activity(), id.clone(), path, size));
1092        }
1093    }
1094    if under_pressure {
1095        // Inventory sizes prioritize scarce-space recovery only; deletion authority
1096        // and budget accounting come from declared outputs measured under locks.
1097        targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1098    } else {
1099        targets.sort_by_key(|t| t.0);
1100    }
1101    let free_before = crate::disk::free_bytes(home).ok();
1102    #[cfg(unix)]
1103    {
1104        let mut seen = std::collections::BTreeSet::new();
1105        report.measured_allocated_file_bytes = Some(0);
1106        report.estimated_released_allocated_bytes = Some(0);
1107        report.proven_reclaimable_allocated_bytes = Some(0);
1108        for (_, _, path, _) in &targets {
1109            report.measured_allocated_file_bytes = report
1110                .measured_allocated_file_bytes
1111                .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1112                .map(|(a, b)| a.saturating_add(b));
1113        }
1114    }
1115    let mut remaining = report.measured_bytes;
1116    let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1117    let mut deletion_started = false;
1118    let mut oversized_started = false;
1119    for (updated, id, path, _) in targets {
1120        if disk.worktree_limit_bytes == 0
1121            || remaining <= disk.worktree_limit_bytes
1122                && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1123        {
1124            report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1125            continue;
1126        }
1127        if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1128            report
1129                .retained
1130                .push(format!("{id}: per-pass byte budget spent"));
1131            continue;
1132        }
1133        let result = async {
1134            ensure!(
1135                crate::clean::due(now, updated, disk.fold_grace_secs),
1136                "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1137            );
1138            let _owner = crate::pr_head::owner_lock(home, &id)?;
1139            let state = RunState::load_under(&id, home)?;
1140            ensure!(
1141                state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1142                    == Liveness::Dead,
1143                "run live or ownership unknown"
1144            );
1145            ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1146            let _driver = crate::driver::maintenance(home, &id)?;
1147            let _outputs = OutputLease::take(&path, true)?;
1148            ensure!(
1149                worktrees(&state).await?.contains(&path),
1150                "worktree registration changed"
1151            );
1152            let target = validate_cargo_target(&path, true).await
1153                .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1154            let _cargo = cargo_locks(&target)
1155                .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1156            // Cargo may have finished between identity inspection and locking.
1157            // Revalidate the same authority with all writers excluded.
1158            validate_cargo_target(&path, true).await
1159                .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1160            let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1161            if lease_meta.len() > 0 {
1162                ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1163                let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1164                ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1165                    "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1166            }
1167            output_age(&target, now, disk.fold_grace_secs)
1168                .with_context(|| format!("stage output-age: {}", target.display()))?;
1169            let mut outputs = declared_outputs(&target)
1170                .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1171            let incremental = match crate::incremental::candidates(&path) {
1172                Ok(candidates) => candidates,
1173                Err(error) => {
1174                    report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1175                    crate::incremental::Reclaim::default()
1176                }
1177            };
1178            let incremental_files = incremental.files.clone();
1179            outputs.extend(incremental_files.iter().cloned());
1180            outputs.sort();
1181            outputs.dedup();
1182            ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1183            let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1184                .with_context(|| format!("stage measure: {}", p.display())))
1185                .collect::<Result<Vec<_>>>()?;
1186            let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1187            report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1188            let mut allocation = ReleasedAllocation::of(&outputs)?;
1189            report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1190                .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1191            let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1192            // 予算は1回の削除量を抑えるもので、大きな出力を永久に保持するためではない。
1193            // 正の予算では最初の削除対象だけ超過を許し、その後は予算消費済みとする。
1194            // 途中で失敗しても再度許可しないよう、例外は削除開始時に消費する。
1195            ensure!(!deletion_started || !exceeds_budget,
1196                "declared outputs exceed remaining per-pass byte budget");
1197            // Record adoption before deletion; interrupted deletion is revalidated
1198            // on the next pass. No source file or branch is modified.
1199            let ledger = home.join("runs").join(&id).join("build-outputs.json");
1200            std::fs::write(
1201                &ledger,
1202                serde_json::to_vec(
1203                    &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1204                )?,
1205            ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1206            // Unknown leftovers still permit artifact-only eviction, but never
1207            // authorize retirement. Only a complete proven residue is adopted.
1208            if !incremental_files.is_empty() {
1209                report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1210            } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1211                report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1212            }
1213            ensure!(worktrees(&state).await?.contains(&path),
1214                "worktree registration changed before deletion");
1215            deletion_started = true;
1216            oversized_started = exceeds_budget;
1217            measurements.remove(&target);
1218            for (output, size) in outputs.iter().zip(sizes) {
1219                std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1220                report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1221                    .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1222                // Preserve actual progress even if a later removal fails.
1223                report.freed_bytes = report.freed_bytes.saturating_add(size);
1224                remaining = remaining.saturating_sub(size);
1225            }
1226            incremental.finish().context("finish incremental reclamation")?;
1227            report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1228            Ok::<_, anyhow::Error>(())
1229        }
1230        .await;
1231        if let Err(e) = result {
1232            report.retained.push(format!(
1233                "{id}/{}: {e:#}",
1234                path.file_name().unwrap_or_default().to_string_lossy()
1235            ));
1236        }
1237    }
1238    measurements.retain(|path, _| observed.contains(path));
1239    if let Ok(body) = serde_json::to_vec(&measurements)
1240        && let Err(e) = std::fs::write(&measurements_path, body)
1241    {
1242        report.failures.push(format!("output measurements: {e}"));
1243    }
1244    copies::evict(home, disk, now, &mut report).await;
1245    retire_empty(home, disk, now, &mut report).await;
1246    report.protected_or_unproven_bytes = report
1247        .measured_bytes
1248        .saturating_sub(report.proven_reclaimable_bytes);
1249    let pressure_remaining = pressure_paths
1250        .iter()
1251        .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1252    if !report.failures.is_empty()
1253        || disk.worktree_limit_bytes > 0
1254            && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1255    {
1256        let details = report
1257            .retained
1258            .iter()
1259            .filter(|reason| {
1260                !reason.contains("within policy")
1261                    && !reason.ends_with("unmanifested files and Cargo metadata retained")
1262            })
1263            .take(4)
1264            .chain(report.failures.iter().take(2))
1265            .cloned()
1266            .collect::<Vec<_>>()
1267            .join("; ");
1268        crate::notices::raise_in(
1269            home,
1270            crate::notices::Notice::warn(
1271                "disk:retention",
1272                format!(
1273                    "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1274                    report.measured_bytes,
1275                    report.proven_reclaimable_bytes,
1276                    report.protected_or_unproven_bytes,
1277                    report.freed_bytes,
1278                    if report.proven_reclaimable_bytes == 0 {
1279                        "No inspected contents have deletion authority."
1280                    } else {
1281                        "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1282                    }
1283                ),
1284            ),
1285        );
1286    }
1287    report.observed_home_free_change_bytes = free_before
1288        .zip(crate::disk::free_bytes(home).ok())
1289        .map(|(before, after)| {
1290            (i128::from(after) - i128::from(before))
1291                .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1292        });
1293    let store = home.join("retention.json");
1294    if let Ok(body) = serde_json::to_vec_pretty(&report) {
1295        let tmp = home.join("retention.json.tmp");
1296        if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1297            tracing::warn!("retention report could not be saved: {e}");
1298        }
1299    }
1300    tracing::info!(
1301        measured_bytes = report.measured_bytes,
1302        freed_bytes = report.freed_bytes,
1303        retired = report.retired,
1304        retained = report.retained.len(),
1305        failures = report.failures.len(),
1306        "retained worktree cleanup"
1307    );
1308    for reason in report.retained.iter().chain(&report.failures) {
1309        tracing::debug!(%reason, "worktree retained");
1310    }
1311    report
1312}
1313
1314async fn retire_empty(
1315    home: &Path,
1316    disk: &crate::config::Disk,
1317    now: Timestamp,
1318    report: &mut Report,
1319) {
1320    if !disk.auto_fold {
1321        return;
1322    }
1323    let queue = crate::queue::Queue::at(home.join("queue"));
1324    let questions = crate::ask::Questions::at(home.join("questions"));
1325    let open = questions.list();
1326    for listed in queue.list() {
1327        let Ok(_claim) = queue.claim(&listed.id) else {
1328            continue;
1329        };
1330        let Ok(task) = queue.get(&listed.id) else {
1331            continue;
1332        };
1333        if open
1334            .iter()
1335            .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1336        {
1337            report.retained.push(format!(
1338                "{}: task has an unanswered owner decision",
1339                task.id
1340            ));
1341            continue;
1342        }
1343        let sources: Vec<_> = task
1344            .runs
1345            .iter()
1346            .collect::<std::collections::BTreeSet<_>>()
1347            .into_iter()
1348            .cloned()
1349            .collect();
1350        let mut useful = Vec::new();
1351        let mut selected = Vec::new();
1352        let mut public = Vec::new();
1353        let mut unreadable = false;
1354        for id in &sources {
1355            let Ok(state) = RunState::load_under(id, home) else {
1356                unreadable = true;
1357                continue;
1358            };
1359            if state.released() || state.retired_to.is_some() {
1360                continue;
1361            }
1362            if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1363                selected.push(state.clone());
1364            }
1365            if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1366                public.push(state.clone());
1367            }
1368            if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1369                useful.push(state);
1370            }
1371        }
1372        // Prefer a durable owner-approved selection, then one existing PR
1373        // owner, then an unambiguous useful source. Distinct historical changes
1374        // without an owner selection cannot be resolved by creation time.
1375        let pool = if !selected.is_empty() {
1376            selected
1377        } else if !public.is_empty() {
1378            public
1379        } else {
1380            useful
1381        };
1382        if unreadable || pool.len() != 1 {
1383            report.retained.push(format!(
1384                "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1385                task.id,
1386                sources.len()
1387            ));
1388            continue;
1389        }
1390        let authority = &pool[0];
1391        if authority.candidates.is_empty() {
1392            continue;
1393        }
1394        let latest = &authority.id;
1395        for id in &sources {
1396            if id == latest || report.retired >= 5 {
1397                continue;
1398            }
1399            if let Ok(state) = RunState::load_under(id, home)
1400                && state.retired_to.as_ref() == Some(latest)
1401                && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1402            {
1403                continue;
1404            }
1405            let result = async {
1406                ensure!(
1407                    !task
1408                        .resume_override
1409                        .as_ref()
1410                        .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1411                    "operator-pinned recovery run"
1412                );
1413                ensure!(
1414                    !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1415                    "selected replacement recovery source"
1416                );
1417                let _owner = crate::pr_head::owner_lock(home, id)?;
1418                let mut state = RunState::load_under(id, home)?;
1419                ensure!(
1420                    state.repo == authority.repo,
1421                    "repository differs from recovery authority"
1422                );
1423                ensure!(
1424                    !crate::pr_recovery::explicit_sources_retained(home, id),
1425                    "explicitly retained recovery source"
1426                );
1427                ensure!(
1428                    state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1429                        == Liveness::Dead,
1430                    "run live or ownership unknown"
1431                );
1432                let _driver = crate::driver::maintenance(home, id)?;
1433                ensure!(
1434                    crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1435                    "within grace period"
1436                );
1437                ensure!(
1438                    !open.iter().any(|q| q.run == *id && q.status.open()),
1439                    "unanswered owner decision"
1440                );
1441                ensure!(
1442                    state.pr.is_none()
1443                        && state.pr_head.is_none()
1444                        && state.pr_recovery.is_none()
1445                        && state.land_approval.is_none()
1446                        && state.merge_decision.is_none()
1447                        && state.reviews.is_empty()
1448                        && state.recovery.is_none()
1449                        && state.candidate_preservation.is_empty()
1450                        && state.interrupted_implementations.is_empty()
1451                        && !state.released(),
1452                    "PR ownership or recovery checkpoint"
1453                );
1454                let proof = prove_empty(home, &state, latest).await?;
1455                commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1456                // Durable retirement precedes removal. A crash retries removal
1457                // using the same proof; it cannot make this attempt resumable.
1458                for path in &proof.paths {
1459                    ensure!(
1460                        git::worktree_remove_clean(&state.repo, path).await?,
1461                        "Git refused clean worktree removal"
1462                    );
1463                }
1464                Ok::<_, anyhow::Error>(())
1465            }
1466            .await;
1467            match result {
1468                Ok(()) => report.retired += 1,
1469                Err(e) => report.retained.push(format!("{id}: {e:#}")),
1470            }
1471        }
1472    }
1473}
1474
1475struct EmptyProof {
1476    paths: Vec<PathBuf>,
1477    _outputs: Vec<OutputLease>,
1478    _cargo: Vec<File>,
1479}
1480
1481// Compare actual source bytes with the original tree independently of index
1482// stat caches, assume-unchanged flags, sparse checkout and fsmonitor hints.
1483pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1484    let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1485    let mut files = Vec::new();
1486    let mut blobs = Vec::new();
1487    for record in tree.split('\0').filter(|s| !s.is_empty()) {
1488        let (header, name) = record.split_once('\t').context("unknown tree record")?;
1489        let parts: Vec<_> = header.split_whitespace().collect();
1490        ensure!(
1491            parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1492            "unsupported tracked source kind"
1493        );
1494        let path = worktree.join(name);
1495        let meta = std::fs::symlink_metadata(&path)?;
1496        ensure!(
1497            meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1498            "noncanonical tracked source"
1499        );
1500        #[cfg(unix)]
1501        {
1502            use std::os::unix::fs::PermissionsExt as _;
1503            ensure!(
1504                (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1505                "tracked executable mode differs from original base"
1506            );
1507        }
1508        path.to_str().context("non UTF-8 source")?;
1509        files.push(git::path_arg(&path));
1510        blobs.push(parts[2].to_owned());
1511    }
1512    for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1513        let mut args = vec!["hash-object", "--no-filters", "--"];
1514        args.extend(paths.iter().map(String::as_str));
1515        let actual = git::git(worktree, &args).await?;
1516        ensure!(
1517            actual.lines().eq(expected.iter().map(String::as_str)),
1518            "tracked bytes differ from original base"
1519        );
1520    }
1521    Ok(())
1522}
1523
1524async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1525    let id = &state.id;
1526    let paths = worktrees(state).await?;
1527    let mut guards = Vec::new();
1528    let mut cargo_guards = Vec::new();
1529    for path in &paths {
1530        guards.push(OutputLease::take(path, true)?);
1531        ensure!(
1532            git::rev_parse(path, "HEAD").await? == state.base_commit,
1533            "unique or unrecorded commits"
1534        );
1535        ensure!(
1536            git::git(
1537                path,
1538                &[
1539                    "--no-optional-locks",
1540                    "status",
1541                    "--porcelain",
1542                    "--untracked-files=all"
1543                ]
1544            )
1545            .await?
1546            .is_empty(),
1547            "tracked or untracked source changes"
1548        );
1549        ensure!(
1550            git::git(
1551                path,
1552                &[
1553                    "--no-optional-locks",
1554                    "status",
1555                    "--porcelain",
1556                    "--untracked-files=all",
1557                    "--ignored",
1558                    "--",
1559                    ".",
1560                    ":(exclude)target"
1561                ]
1562            )
1563            .await?
1564            .is_empty(),
1565            "source changes, ignored files or build outputs"
1566        );
1567        tracked_base(path, &state.base_commit).await?;
1568        match std::fs::symlink_metadata(path.join("target")) {
1569            Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1570            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1571            Err(error) => return Err(error.into()),
1572        }
1573        for checkpoint in [
1574            "rebase-merge",
1575            "rebase-apply",
1576            "sequencer",
1577            "BISECT_LOG",
1578            "REVERT_HEAD",
1579            "AUTO_MERGE",
1580            "MERGE_MSG",
1581            "CHERRY_PICK_HEAD",
1582            "MERGE_HEAD",
1583            "REBASE_HEAD",
1584        ] {
1585            let location = git::git(
1586                path,
1587                &[
1588                    "rev-parse",
1589                    "--path-format=absolute",
1590                    "--git-path",
1591                    checkpoint,
1592                ],
1593            )
1594            .await?;
1595            ensure!(
1596                matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1597                "retained Git recovery metadata: {checkpoint}"
1598            );
1599        }
1600        // Git also initializes ORIG_HEAD when creating a worktree.
1601        // Only an origin outside the base is a recovery source.
1602        let origin_path = git::git(
1603            path,
1604            &[
1605                "rev-parse",
1606                "--path-format=absolute",
1607                "--git-path",
1608                "ORIG_HEAD",
1609            ],
1610        )
1611        .await?;
1612        match std::fs::symlink_metadata(origin_path.trim()) {
1613            Ok(meta) => {
1614                ensure!(
1615                    meta.is_file() && !meta.file_type().is_symlink(),
1616                    "unknown ORIG_HEAD artifact"
1617                );
1618                git::rev_parse(path, "ORIG_HEAD")
1619                    .await
1620                    .context("unreadable ORIG_HEAD")?;
1621            }
1622            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1623            Err(e) => return Err(e.into()),
1624        }
1625        if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1626            ensure!(
1627                git::git_raw(
1628                    path,
1629                    &["merge-base", "--is-ancestor", &original, &state.base_commit]
1630                )
1631                .await?
1632                .ok(),
1633                "ORIG_HEAD retains commits outside the original base"
1634            );
1635        }
1636        let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1637        for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1638            ensure!(
1639                git::git_raw(
1640                    path,
1641                    &["merge-base", "--is-ancestor", head, &state.base_commit]
1642                )
1643                .await?
1644                .ok(),
1645                "worktree reflog retains commits outside the original base"
1646            );
1647        }
1648        for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1649            ensure!(
1650                !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1651                    .await?
1652                    .ok(),
1653                "unfinished Git operation"
1654            );
1655        }
1656    }
1657    // Missing worktrees cannot establish that unrecorded prep edits
1658    // were empty. Recorded candidates must also point at the base.
1659    if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1660        return Ok(EmptyProof {
1661            paths,
1662            _outputs: guards,
1663            _cargo: cargo_guards,
1664        });
1665    }
1666    ensure!(!paths.is_empty(), "no worktrees to prove empty");
1667    for candidate in &state.candidates {
1668        ensure!(
1669            (paths.contains(&candidate.worktree)
1670                || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1671                && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1672            "candidate recovery source"
1673        );
1674    }
1675    let pattern = format!("refs/heads/magi/{}/", state.short());
1676    let branches = git::git(
1677        &state.repo,
1678        &["for-each-ref", "--format=%(objectname)", &pattern],
1679    )
1680    .await?;
1681    ensure!(
1682        branches.lines().all(|head| head == state.base_commit),
1683        "unrecorded candidate branch changes"
1684    );
1685    Ok(EmptyProof {
1686        paths,
1687        _outputs: guards,
1688        _cargo: cargo_guards,
1689    })
1690}
1691
1692/// The only task-write critical section in retirement. All Git proof and
1693/// deletion happens outside it, under the run's owner/driver/output leases.
1694/// Atomic task writes make the earlier snapshot consistent; this final compare
1695/// rejects every intervening owner selection, hold, edit or history update.
1696fn commit_retirement(
1697    home: &Path,
1698    queue: &crate::queue::Queue,
1699    expected: &crate::queue::Task,
1700    state: &mut RunState,
1701    authority: &str,
1702    sources: &[String],
1703    now: Timestamp,
1704) -> Result<()> {
1705    let _write = queue.lock_task(&expected.id)?;
1706    let current = queue.get(&expected.id)?;
1707    ensure!(
1708        serde_json::to_value(&current)? == serde_json::to_value(expected)?,
1709        "task changed during retirement proof; retaining worktrees for the current owner decision"
1710    );
1711    ensure!(
1712        !current
1713            .resume_override
1714            .as_ref()
1715            .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1716            && !current
1717                .replacement
1718                .as_ref()
1719                .is_some_and(|pin| pin.run == state.id),
1720        "current owner selected this recovery source"
1721    );
1722    ensure!(
1723        !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1724        "explicitly retained recovery source"
1725    );
1726    ensure!(
1727        !crate::ask::Questions::at(home.join("questions"))
1728            .list()
1729            .iter()
1730            .any(|q| q.status.open()
1731                && (q.run == state.id
1732                    || q.run == current.id
1733                    || current.blocked_by.contains(&q.id))),
1734        "owner decision arrived during retirement proof"
1735    );
1736    let dir = home.join("recovery-lineages");
1737    std::fs::create_dir_all(&dir)?;
1738    let tmp = dir.join(format!("{}.tmp", current.id));
1739    let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1740        "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1741    std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1742    std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1743    state.retired_to = Some(authority.to_owned());
1744    state.event(
1745        "retention",
1746        format!(
1747            "empty attempt retired to {authority}; recovery sources: {}",
1748            sources.join(", ")
1749        ),
1750    );
1751    state.save_under(home)?;
1752    // No Git command or worktree removal may be added before this guard drops.
1753    Ok(())
1754}
1755
1756/// Wait for the emergency reserve to be exhausted. The normal floor requests
1757/// a checkpoint first; half the floor stops owned writers before node completion.
1758/// A failed measurement also closes admission rather than allowing blind writes.
1759pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1760    if min_free == 0 {
1761        return std::future::pending().await;
1762    }
1763    loop {
1764        for path in &paths {
1765            let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1766            match crate::disk::free_bytes(existing) {
1767                Ok(free) if free >= min_free / 2 => {}
1768                Ok(free) => {
1769                    return format!(
1770                        "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1771                    );
1772                }
1773                Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1774            }
1775        }
1776        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1777    }
1778}
1779
1780#[cfg(all(test, unix))]
1781mod tests {
1782    use super::*;
1783    use crate::{
1784        config::{Config, Disk},
1785        queue::{Source, Task},
1786        run::{Candidate, RunStatus},
1787    };
1788
1789    struct Fx {
1790        _temp: tempfile::TempDir,
1791        home: PathBuf,
1792        repo: PathBuf,
1793        bay: PathBuf,
1794        base: String,
1795    }
1796    impl Fx {
1797        async fn new() -> Self {
1798            let temp = tempfile::tempdir().unwrap();
1799            let root = temp.path().canonicalize().unwrap();
1800            let home = root.join("home");
1801            let repo = root.join("repo");
1802            let bay = root.join("worktrees");
1803            std::fs::create_dir_all(&repo).unwrap();
1804            std::fs::create_dir_all(home.join("runs")).unwrap();
1805            git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1806            git::git(&repo, &["config", "user.name", "Fixture"])
1807                .await
1808                .unwrap();
1809            git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1810                .await
1811                .unwrap();
1812            git::git(&repo, &["config", "commit.gpgsign", "false"])
1813                .await
1814                .unwrap();
1815            std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1816            std::fs::write(repo.join("source"), "base\n").unwrap();
1817            git::git(&repo, &["add", "."]).await.unwrap();
1818            git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1819            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1820            Self {
1821                _temp: temp,
1822                home,
1823                repo,
1824                bay,
1825                base,
1826            }
1827        }
1828        async fn run(&self, tag: &str, useful: bool) -> RunState {
1829            let mut cfg = Config::default();
1830            cfg.graph.worktree_root = Some(self.bay.clone());
1831            let mut state = RunState::new(
1832                self.repo.clone(),
1833                "main".into(),
1834                self.base.clone(),
1835                "task".into(),
1836                cfg,
1837            );
1838            state.id = format!("20260101-000000-{tag}");
1839            state.status = RunStatus::Blocked;
1840            state.driver_exited = true;
1841            let wt = state.worktree_root().join("cand-A");
1842            let branch = format!("magi/{tag}/A");
1843            git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1844                .await
1845                .unwrap();
1846            if useful {
1847                std::fs::write(wt.join("source"), tag).unwrap();
1848                git::git(&wt, &["add", "."]).await.unwrap();
1849                git::git(&wt, &["commit", "-m", "candidate change"])
1850                    .await
1851                    .unwrap();
1852            }
1853            state.candidates.push(Candidate {
1854                index: 0,
1855                label: 'A',
1856                agent: "fixture".into(),
1857                branch,
1858                worktree: wt,
1859                summary: String::new(),
1860                stat: String::new(),
1861                files: usize::from(useful),
1862                commits: usize::from(useful),
1863                empty: !useful,
1864                failed: None,
1865                verified_noop: None,
1866                duration_ms: 0,
1867                folded: false,
1868            });
1869            state.save_under(&self.home).unwrap();
1870            state
1871        }
1872        fn disk() -> Disk {
1873            Disk {
1874                worktree_limit_bytes: 1,
1875                worktree_reclaim_bytes: 1024 * 1024,
1876                fold_grace_secs: 0,
1877                min_free_bytes: 0,
1878                ..Disk::default()
1879            }
1880        }
1881        fn target(wt: &Path) {
1882            let target = wt.join("target");
1883            std::fs::create_dir_all(target.join("debug")).unwrap();
1884            let mut artifact = [1; 200];
1885            artifact[..4].copy_from_slice(b"\x7fELF");
1886            std::fs::write(target.join("debug/output"), artifact).unwrap();
1887            let output = target
1888                .join("debug/output")
1889                .display()
1890                .to_string()
1891                .replace('\\', "\\\\")
1892                .replace(' ', "\\ ");
1893            std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1894            std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1895            std::fs::write(
1896                target.join("CACHEDIR.TAG"),
1897                "Signature: 8a477f597d28d172789f06886806bc55\n",
1898            )
1899            .unwrap();
1900            std::fs::write(
1901                target.join(".rustc_info.json"),
1902                "{\"rustc_fingerprint\":123}",
1903            )
1904            .unwrap();
1905        }
1906        fn now() -> Timestamp {
1907            Timestamp::now() + jiff::SignedDuration::from_secs(2)
1908        }
1909        fn task(&self, runs: &[RunState]) -> Task {
1910            let mut task = Task::new(
1911                "fixture".into(),
1912                "task".into(),
1913                self.repo.clone(),
1914                Source::Human,
1915            );
1916            task.runs = runs.iter().map(|s| s.id.clone()).collect();
1917            crate::queue::Queue::at(self.home.join("queue"))
1918                .put(&mut task)
1919                .unwrap();
1920            task
1921        }
1922    }
1923
1924    #[cfg(unix)]
1925    #[tokio::test]
1926    async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1927        let f = Fx::new().await;
1928        let state = f.run("ca01", true).await;
1929        let wt = &state.candidates[0].worktree;
1930        let generation = crate::incremental::tests::generate(wt);
1931        // Remove dep-info authority: incremental proof must be independent.
1932        for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1933            for entry in std::fs::read_dir(dir).unwrap() {
1934                let path = entry.unwrap().path();
1935                if path.extension().is_some_and(|x| x == "d") {
1936                    std::fs::remove_file(path).unwrap();
1937                }
1938            }
1939        }
1940        assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1941        assert_eq!(
1942            sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1943            0,
1944            "live generation is protected"
1945        );
1946        drop(generation);
1947        let candidates = crate::incremental::candidates(wt).unwrap();
1948        let logical: u64 = candidates
1949            .files
1950            .iter()
1951            .map(|p| std::fs::metadata(p).unwrap().len())
1952            .sum();
1953        let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1954        std::fs::write(&unknown, "retain unknown data").unwrap();
1955        drop(candidates);
1956        let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1957        assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1958        assert!(
1959            refused
1960                .retained
1961                .iter()
1962                .any(|s| s.contains("provenance mismatch"))
1963        );
1964        std::fs::remove_file(&unknown).unwrap();
1965        let legacy = wt.join("target/debug/incremental/legacy-session");
1966        std::fs::create_dir(&legacy).unwrap();
1967        // Even compiler-authored bytes do not retroactively confer ownership.
1968        let candidates = crate::incremental::candidates(wt).unwrap();
1969        let bin = candidates
1970            .files
1971            .iter()
1972            .find(|p| p.extension().is_some_and(|x| x == "bin"))
1973            .unwrap();
1974        std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1975        drop(candidates);
1976        let cargo = cargo_locks(&wt.join("target")).unwrap();
1977        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1978        drop(cargo);
1979        let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1980        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1981        drop(outputs);
1982        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1983        assert_eq!(report.freed_bytes, logical, "{report:?}");
1984        assert_eq!(report.proven_reclaimable_bytes, logical);
1985        assert!(report.measured_bytes > logical);
1986        assert_eq!(
1987            report.protected_or_unproven_bytes,
1988            report.measured_bytes - logical
1989        );
1990        #[cfg(unix)]
1991        {
1992            assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
1993            assert!(
1994                report.measured_allocated_file_bytes.unwrap()
1995                    >= report.estimated_released_allocated_bytes.unwrap()
1996            );
1997        }
1998        assert!(report.observed_home_free_change_bytes.is_some());
1999        assert!(legacy.join("query-cache.bin").is_file());
2000        assert!(wt.join("src/lib.rs").is_file());
2001        assert_eq!(report.retired, 0);
2002    }
2003
2004    #[cfg(unix)]
2005    #[tokio::test]
2006    async fn incremental_and_deps_links_account_for_retained_third_link() {
2007        use std::os::unix::fs::MetadataExt;
2008        let f = Fx::new().await;
2009        let state = f.run("ca02", true).await;
2010        let wt = &state.candidates[0].worktree;
2011        let generation = crate::incremental::tests::generate(wt);
2012        drop(generation);
2013        let candidates = crate::incremental::candidates(wt).unwrap();
2014        let objects = candidates
2015            .files
2016            .iter()
2017            .filter(|p| p.extension().is_some_and(|x| x == "o"))
2018            .take(2)
2019            .cloned()
2020            .collect::<Vec<_>>();
2021        assert_eq!(
2022            objects.len(),
2023            2,
2024            "fixture must produce real incremental objects"
2025        );
2026        let mut outputs = candidates.files.clone();
2027        drop(candidates);
2028        let target = wt.join("target");
2029        for dir in [target.join("debug"), target.join("debug/deps")] {
2030            for entry in std::fs::read_dir(dir).unwrap() {
2031                let path = entry.unwrap().path();
2032                if path.extension().is_some_and(|x| x == "d") {
2033                    std::fs::remove_file(path).unwrap();
2034                }
2035            }
2036        }
2037        // Cargo also leaves hard-linked object outputs in deps. Remove those
2038        // fixture outputs so the only external links are the ones below.
2039        for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
2040            let path = entry.unwrap().path();
2041            if path.is_file() {
2042                std::fs::remove_file(path).unwrap();
2043            }
2044        }
2045        for (index, object) in objects.iter().enumerate() {
2046            assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2047            let dep = target.join(format!("debug/deps/link{index}"));
2048            std::fs::hard_link(object, &dep).unwrap();
2049            std::fs::write(
2050                dep.with_extension("d"),
2051                format!("{}: src/lib.rs\n", dep.display()),
2052            )
2053            .unwrap();
2054            outputs.push(dep);
2055        }
2056        let retained = wt.join("retained-input");
2057        std::fs::hard_link(&objects[1], &retained).unwrap();
2058        let sentinel = std::fs::read(&retained).unwrap();
2059        let logical: u64 = outputs
2060            .iter()
2061            .map(|p| std::fs::metadata(p).unwrap().len())
2062            .sum();
2063        let mut seen = std::collections::BTreeSet::new();
2064        let retained_meta = std::fs::metadata(&retained).unwrap();
2065        let retained_inode = (retained_meta.dev(), retained_meta.ino());
2066        let mut expected = 0u64;
2067        for output in &outputs {
2068            let m = std::fs::metadata(output).unwrap();
2069            let inode = (m.dev(), m.ino());
2070            if inode != retained_inode && seen.insert(inode) {
2071                expected += m.blocks() * 512;
2072            }
2073        }
2074        assert!(expected > 0);
2075        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2076        assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2077        assert_eq!(report.freed_bytes, logical, "{report:?}");
2078        assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2079        assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2080        assert!(outputs.iter().all(|p| !p.exists()));
2081        assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2082        assert!(wt.join("src/lib.rs").is_file());
2083    }
2084
2085    #[tokio::test]
2086    async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2087        let f = Fx::new().await;
2088        let state = f.run("ca03", true).await;
2089        let wt = &state.candidates[0].worktree;
2090        Fx::target(wt);
2091        std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2092        let unknown = wt.join("target/debug/operator-note");
2093        std::fs::write(&unknown, vec![1; 4096]).unwrap();
2094        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2095        assert_eq!(report.proven_reclaimable_bytes, 0);
2096        #[cfg(unix)]
2097        assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2098        assert!(report.protected_or_unproven_bytes >= 4096);
2099        let message = || {
2100            crate::notices::Notices::at(f.home.join("notifications"))
2101                .list()
2102                .into_iter()
2103                .find(|n| n.key == "disk:retention")
2104                .unwrap()
2105                .message
2106        };
2107        let notice = message();
2108        assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2109        assert!(notice.contains("protected or unproven"), "{notice}");
2110        assert!(
2111            notice.contains("legacy incremental/build and unknown files retained"),
2112            "{notice}"
2113        );
2114        let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2115        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2116        assert_eq!(report.freed_bytes, 0);
2117        let notice = message();
2118        assert!(notice.contains(&state.id), "{notice}");
2119        assert!(
2120            notice.contains("lock") || notice.contains("owner"),
2121            "{notice}"
2122        );
2123        drop(owner);
2124        assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2125    }
2126
2127    #[tokio::test]
2128    async fn real_cargo_build_holds_the_target_against_reclamation() {
2129        let f = Fx::new().await;
2130        let state = f.run("ca02", true).await;
2131        let wt = &state.candidates[0].worktree;
2132        Fx::target(wt);
2133        std::fs::create_dir(wt.join("src")).unwrap();
2134        std::fs::write(
2135            wt.join("Cargo.toml"),
2136            "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2137        )
2138        .unwrap();
2139        std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2140        std::fs::write(
2141            wt.join("build.rs"),
2142            r#"fn main() {
2143            let ready = std::env::var("FIXTURE_READY").unwrap();
2144            let release = std::env::var("FIXTURE_RELEASE").unwrap();
2145            std::fs::write(ready, "ready").unwrap();
2146            let start = std::time::Instant::now();
2147            while !std::path::Path::new(&release).exists() && start.elapsed().as_secs() < 15 {
2148                std::thread::sleep(std::time::Duration::from_millis(10));
2149            }
2150        }"#,
2151        )
2152        .unwrap();
2153        let ready = wt.join("ready");
2154        let release = wt.join("release");
2155        let mut cargo = tokio::process::Command::new("cargo");
2156        cargo.env_clear();
2157        for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2158            if let Some(value) = std::env::var_os(key) {
2159                cargo.env(key, value);
2160            }
2161        }
2162        let mut child = cargo
2163            .current_dir(wt)
2164            .env("RUSTUP_TOOLCHAIN", "stable")
2165            .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2166            .env("CARGO_INCREMENTAL", "1")
2167            .env("FIXTURE_READY", &ready)
2168            .env("FIXTURE_RELEASE", &release)
2169            .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2170            .arg(wt.join("target"))
2171            .stdout(std::process::Stdio::null())
2172            .stderr(std::process::Stdio::null())
2173            .kill_on_drop(true)
2174            .spawn()
2175            .unwrap();
2176        let arrived = tokio::time::timeout(std::time::Duration::from_secs(10), async {
2177            while !ready.exists() {
2178                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2179            }
2180        })
2181        .await
2182        .is_ok();
2183        let report = if arrived {
2184            Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2185        } else {
2186            None
2187        };
2188        // Release and join our build before making assertions, including on failure.
2189        std::fs::write(release, "release").unwrap();
2190        let status = tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await;
2191        if status.is_err() {
2192            child.kill().await.unwrap();
2193            child.wait().await.unwrap();
2194        }
2195        assert!(
2196            arrived,
2197            "fixture build never reached its active build script"
2198        );
2199        assert!(status.unwrap().unwrap().success());
2200        let report = report.unwrap();
2201        assert_eq!(report.freed_bytes, 0, "{report:?}");
2202        assert!(
2203            report
2204                .retained
2205                .iter()
2206                .any(|s| s.contains("Cargo build is active")),
2207            "{report:?}"
2208        );
2209        assert!(wt.join("target/debug/output").is_file());
2210        assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2211    }
2212
2213    #[cfg(unix)]
2214    #[test]
2215    fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2216        let temp = tempfile::tempdir().unwrap();
2217        let a = temp.path().join("a");
2218        let b = temp.path().join("b");
2219        let outside = temp.path().join("retained-input");
2220        std::fs::write(&a, vec![1; 65536]).unwrap();
2221        std::fs::hard_link(&a, &b).unwrap();
2222        std::fs::hard_link(&a, &outside).unwrap();
2223        let mut seen = Default::default();
2224        let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2225        assert!(blocks > 0);
2226        let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2227        std::fs::remove_file(&a).unwrap();
2228        assert_eq!(accounting.unlinked(&a), Some(0));
2229        std::fs::remove_file(&b).unwrap();
2230        assert_eq!(accounting.unlinked(&b), Some(0));
2231        assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2232        let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2233        std::fs::remove_file(&outside).unwrap();
2234        assert_eq!(accounting.unlinked(&outside), Some(blocks));
2235    }
2236
2237    #[cfg(unix)]
2238    #[test]
2239    fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2240        let temp = tempfile::tempdir_in(".").unwrap();
2241        let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2242        assert!(home.is_relative());
2243        let paths = vec![std::env::current_dir().unwrap()];
2244        let gib = 1024 * 1024 * 1024;
2245        let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2246        assert!(
2247            OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2248                9 * gib
2249            ))
2250            .is_err()
2251        );
2252        drop(reservation);
2253        assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2254        let bad = temp.path().join("symlink-state");
2255        std::fs::create_dir(&bad).unwrap();
2256        std::os::unix::fs::symlink(
2257            home.canonicalize().unwrap().join("disk-admission"),
2258            bad.join("disk-admission"),
2259        )
2260        .unwrap();
2261        assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2262    }
2263
2264    #[cfg(unix)]
2265    #[test]
2266    fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2267        let temp = tempfile::tempdir().unwrap();
2268        let home = temp.path().canonicalize().unwrap();
2269        let paths = vec![home.clone(), home.join("not-created-yet")];
2270        let gib = 1024 * 1024 * 1024;
2271        let floor = 8 * gib;
2272        let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2273        assert_eq!(first.len(), 1);
2274        let blocked =
2275            OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2276        assert!(blocked.to_string().contains("active writer reservations"));
2277        let second =
2278            OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2279        drop(first);
2280        assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2281        drop(second);
2282        assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2283        assert!(
2284            OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2285        );
2286    }
2287
2288    #[cfg(unix)]
2289    #[test]
2290    fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2291        let temp = tempfile::tempdir().unwrap();
2292        let home = temp.path().canonicalize().unwrap();
2293        let barrier = std::sync::Barrier::new(2);
2294        let gib = 1024 * 1024 * 1024;
2295        let results = std::thread::scope(|scope| {
2296            let threads: Vec<_> = (0..2)
2297                .map(|_| {
2298                    scope.spawn(|| {
2299                        barrier.wait();
2300                        OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2301                            Ok(100 + gib)
2302                        })
2303                    })
2304                })
2305                .collect();
2306            threads
2307                .into_iter()
2308                .map(|thread| thread.join().unwrap())
2309                .collect::<Vec<_>>()
2310        });
2311        assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2312        assert!(
2313            results
2314                .iter()
2315                .filter_map(|r| r.as_ref().err())
2316                .all(|e| e.to_string().contains("active writer reservations"))
2317        );
2318    }
2319
2320    #[tokio::test]
2321    async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2322        real_cargo_fixture(false).await;
2323    }
2324
2325    #[tokio::test]
2326    async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2327        real_cargo_fixture(true).await;
2328    }
2329
2330    async fn real_cargo_fixture(precreate: bool) {
2331        let f = Fx::new().await;
2332        let state = f.run("ca00", true).await;
2333        let wt = &state.candidates[0].worktree;
2334        let source = "fn main() { println!(\"retained fixture\"); }\n";
2335        std::fs::create_dir(wt.join("src")).unwrap();
2336        std::fs::write(wt.join("src/main.rs"), source).unwrap();
2337        std::fs::write(
2338            wt.join("Cargo.toml"),
2339            "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2340        )
2341        .unwrap();
2342        if precreate {
2343            std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2344        }
2345        let mut cargo = std::process::Command::new("cargo");
2346        cargo.env_clear();
2347        for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2348            if let Some(value) = std::env::var_os(key) {
2349                cargo.env(key, value);
2350            }
2351        }
2352        let build = cargo
2353            .current_dir(wt)
2354            .env("RUSTUP_TOOLCHAIN", "stable")
2355            .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2356            .env("CARGO_INCREMENTAL", "0")
2357            .args([
2358                "build",
2359                "--offline",
2360                "--release",
2361                "--jobs",
2362                "1",
2363                "--target-dir",
2364            ])
2365            .arg(wt.join("target"))
2366            .output()
2367            .unwrap();
2368        assert!(
2369            build.status.success(),
2370            "{}",
2371            String::from_utf8_lossy(&build.stderr)
2372        );
2373        assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2374        if precreate {
2375            // Cargo leaves pre-existing target roots untagged on subsequent builds too.
2376            let again = cargo.output().unwrap();
2377            assert!(
2378                again.status.success(),
2379                "{}",
2380                String::from_utf8_lossy(&again.stderr)
2381            );
2382            assert!(!wt.join("target/CACHEDIR.TAG").exists());
2383            assert!(cargo_target(wt).await.is_err());
2384        }
2385        git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2386            .await
2387            .unwrap();
2388        git::git(wt, &["commit", "-m", "real Cargo fixture"])
2389            .await
2390            .unwrap();
2391        let artifact = wt.join("target/release/retained-fixture");
2392        assert!(artifact.is_file());
2393        let unknown = wt.join("target/release/operator-note");
2394        std::fs::write(&unknown, "keep this note\n").unwrap();
2395
2396        let target = wt.join("target");
2397        let mut preserved = std::collections::BTreeMap::new();
2398        for relative in [
2399            "tmp/sentinel",
2400            "magi-tools/sentinel",
2401            "lifecycle-client-check.log",
2402        ] {
2403            let path = target.join(relative);
2404            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2405            std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2406            preserved.insert(path.clone(), std::fs::read(path).unwrap());
2407        }
2408        // Even genuine compiler bytes and valid dep-info outside supported
2409        // profiles cannot expand the deletion inventory.
2410        let decoy = target.join("magi-tools/retained-fixture");
2411        std::fs::copy(&artifact, &decoy).unwrap();
2412        let dep_info = decoy.with_extension("d");
2413        std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2414        for path in [
2415            decoy,
2416            dep_info,
2417            wt.join("Cargo.toml"),
2418            wt.join("Cargo.lock"),
2419        ] {
2420            preserved.insert(path.clone(), std::fs::read(path).unwrap());
2421        }
2422        let proven = declared_outputs(&target).unwrap();
2423
2424        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2425        assert!(out.freed_bytes > 0, "{out:?}");
2426        for (path, bytes) in preserved {
2427            assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2428        }
2429        assert!(!artifact.exists(), "{out:?}");
2430        assert_eq!(
2431            std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2432            source
2433        );
2434        assert_eq!(
2435            std::fs::read_to_string(&unknown).unwrap(),
2436            "keep this note\n"
2437        );
2438        assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2439        let ledger: serde_json::Value = serde_json::from_slice(
2440            &std::fs::read(
2441                f.home
2442                    .join("runs")
2443                    .join(&state.id)
2444                    .join("build-outputs.json"),
2445            )
2446            .unwrap(),
2447        )
2448        .unwrap();
2449        assert_eq!(
2450            ledger["identity"],
2451            if precreate {
2452                "rustc-profile-dep-info"
2453            } else {
2454                "cache-tag"
2455            }
2456        );
2457        if precreate {
2458            assert!(!target.join("CACHEDIR.TAG").exists());
2459            assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2460        }
2461        let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2462        assert!(deleted.contains(&artifact));
2463        assert_eq!(deleted, proven);
2464        for path in deleted {
2465            assert!(!path.exists());
2466            assert!(
2467                [target.join("release"), target.join("release/deps")]
2468                    .contains(&path.parent().unwrap().to_owned())
2469            );
2470        }
2471        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2472    }
2473
2474    fn missing_tag_target(wt: &Path) {
2475        Fx::target(wt);
2476        let target = wt.join("target");
2477        std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2478        for subdir in ["debug/deps", "debug/.fingerprint"] {
2479            std::fs::create_dir_all(target.join(subdir)).unwrap();
2480        }
2481        std::fs::write(target.join(".rustc_info.json"),
2482            r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2483    }
2484
2485    #[tokio::test]
2486    async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2487        for legacy in [false, true] {
2488            let f = Fx::new().await;
2489            let mut state = f.run("ac00", true).await;
2490            Fx::target(&state.candidates[0].worktree);
2491            let original = state.updated_at;
2492            let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2493            if legacy {
2494                let path = f.home.join("runs").join(&state.id).join("run.json");
2495                let mut value = serde_json::to_value(&state).unwrap();
2496                value.as_object_mut().unwrap().remove("output_activity_at");
2497                std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2498                state = RunState::load_under(&state.id, &f.home).unwrap();
2499                assert_eq!(state.output_activity(), original);
2500            }
2501            let age = state.output_activity();
2502            for offset in [30, 20, 10] {
2503                state
2504                    .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2505                    .unwrap();
2506                state = RunState::load_under(&state.id, &f.home).unwrap();
2507                assert_eq!(state.output_activity(), age);
2508            }
2509            let mut disk = Fx::disk();
2510            disk.fold_grace_secs = 3600;
2511            let report = sweep(&f.home, &disk, now).await;
2512            assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2513        }
2514    }
2515
2516    #[tokio::test]
2517    async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2518        for evidence in [
2519            "execution",
2520            "artifact",
2521            "dep-info",
2522            "fingerprint",
2523            "lease",
2524            "future",
2525        ] {
2526            let f = Fx::new().await;
2527            let mut state = f.run("ac01", true).await;
2528            let wt = state.candidates[0].worktree.clone();
2529            missing_tag_target(&wt);
2530            let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2531            let recent = now - jiff::SignedDuration::from_secs(10);
2532            match evidence {
2533                "execution" => {
2534                    state.note_output_activity(recent);
2535                    state.save_under_at(&f.home, recent).unwrap();
2536                }
2537                "lease" => {
2538                    let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2539                    let file = lease._file.try_clone().unwrap();
2540                    drop(lease);
2541                    file.set_modified(std::time::SystemTime::from(recent))
2542                        .unwrap();
2543                }
2544                _ => {
2545                    let relative = match evidence {
2546                        "dep-info" => "debug/output.d",
2547                        "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2548                        _ => "debug/output",
2549                    };
2550                    let path = wt.join("target").join(relative);
2551                    let file = OpenOptions::new()
2552                        .write(true)
2553                        .create(true)
2554                        .truncate(false)
2555                        .open(path)
2556                        .unwrap();
2557                    let at = if evidence == "future" {
2558                        now + jiff::SignedDuration::from_secs(10)
2559                    } else {
2560                        recent
2561                    };
2562                    file.set_modified(std::time::SystemTime::from(at)).unwrap();
2563                }
2564            }
2565            let mut disk = Fx::disk();
2566            disk.fold_grace_secs = 3600;
2567            let report = sweep(&f.home, &disk, now).await;
2568            assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2569            assert!(
2570                report.retained.iter().any(|s| s.contains("grace period")
2571                    || s.contains("execution activity after inventory")),
2572                "{evidence}: {report:?}"
2573            );
2574            let aged = now + jiff::SignedDuration::from_secs(7200);
2575            let report = sweep(&f.home, &disk, aged).await;
2576            assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2577        }
2578    }
2579
2580    #[tokio::test]
2581    async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2582        let f = Fx::new().await;
2583        let state = f.run("ac02", true).await;
2584        let wt = &state.candidates[0].worktree;
2585        missing_tag_target(wt);
2586        let target = wt.join("target");
2587        let unknown = target.join("debug/notes");
2588        std::fs::write(&unknown, "operator data").unwrap();
2589        for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2590            let path = target.join("debug").join(lock);
2591            let file = OpenOptions::new()
2592                .create(true)
2593                .truncate(false)
2594                .read(true)
2595                .write(true)
2596                .open(&path)
2597                .unwrap();
2598            crate::pr_head::try_lock_file(&file).unwrap();
2599            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2600            assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2601            assert!(
2602                report
2603                    .retained
2604                    .iter()
2605                    .any(|s| s.contains(path.to_str().unwrap())),
2606                "{report:?}"
2607            );
2608        }
2609        let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2610        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2611        drop(owner);
2612        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2613        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2614        drop(driver);
2615        let output = OutputLease::borrow(wt).unwrap().unwrap();
2616        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2617        drop(output);
2618        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2619        assert_eq!(report.freed_bytes, 200, "{report:?}");
2620        assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2621        assert!(!target.join("CACHEDIR.TAG").exists());
2622        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2623        assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2624        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2625    }
2626
2627    #[tokio::test]
2628    async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2629        let f = Fx::new().await;
2630        let state = f.run("ac04", true).await;
2631        let wt = &state.candidates[0].worktree;
2632        missing_tag_target(wt);
2633        let target = wt.join("target");
2634        let first = target.join("debug/output");
2635        let second = target.join("debug/second");
2636        std::fs::copy(&first, &second).unwrap();
2637        std::fs::write(
2638            second.with_extension("d"),
2639            format!("{}: source\n", second.display()),
2640        )
2641        .unwrap();
2642        let outputs = declared_outputs(&target).unwrap();
2643        assert_eq!(outputs.len(), 2);
2644        // Simulate interruption after durable adoption and one successful unlink.
2645        std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2646            serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2647        std::fs::remove_file(first).unwrap();
2648        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2649        assert_eq!(report.freed_bytes, 200, "{report:?}");
2650        assert!(!second.exists());
2651        assert!(second.with_extension("d").exists());
2652        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2653    }
2654
2655    #[tokio::test]
2656    async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2657        for invalid in [
2658            "identity",
2659            "deps",
2660            "fingerprint",
2661            "lock",
2662            "dep-info",
2663            "tag-symlink",
2664            "bad-tag",
2665        ] {
2666            let f = Fx::new().await;
2667            let state = f.run("ac03", true).await;
2668            let wt = &state.candidates[0].worktree;
2669            missing_tag_target(wt);
2670            let target = wt.join("target");
2671            match invalid {
2672                "identity" => std::fs::write(
2673                    target.join(".rustc_info.json"),
2674                    r#"{"rustc_fingerprint":123}"#,
2675                )
2676                .unwrap(),
2677                "deps" | "fingerprint" => {
2678                    let path = target.join("debug").join(if invalid == "deps" {
2679                        "deps"
2680                    } else {
2681                        ".fingerprint"
2682                    });
2683                    std::fs::remove_dir(&path).unwrap();
2684                    std::os::unix::fs::symlink(&f.repo, path).unwrap();
2685                }
2686                "lock" => {
2687                    std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2688                }
2689                "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2690                "tag-symlink" => {
2691                    std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2692                        .unwrap()
2693                }
2694                "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2695                _ => unreachable!(),
2696            }
2697            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2698            assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2699            assert!(target.join("debug/output").is_file());
2700        }
2701    }
2702
2703    #[tokio::test]
2704    async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2705        let f = Fx::new().await;
2706        let first = f.run("ba00", true).await;
2707        let second = f.run("ba01", true).await;
2708        for state in [&first, &second] {
2709            Fx::target(&state.candidates[0].worktree);
2710        }
2711        let mut disk = Fx::disk();
2712        disk.worktree_reclaim_bytes = 100;
2713        let out = sweep(&f.home, &disk, Fx::now()).await;
2714        assert_eq!(out.freed_bytes, 200, "{out:?}");
2715        assert!(
2716            out.retained
2717                .contains(&format!("{}: per-pass byte budget spent", second.id)),
2718            "{out:?}"
2719        );
2720        assert!(
2721            !first.candidates[0]
2722                .worktree
2723                .join("target/debug/output")
2724                .exists()
2725        );
2726        assert!(
2727            second.candidates[0]
2728                .worktree
2729                .join("target/debug/output")
2730                .exists()
2731        );
2732        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2733    }
2734
2735    #[tokio::test]
2736    async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2737        let f = Fx::new().await;
2738        let small = f.run("bb00", true).await;
2739        let large = f.run("bb01", true).await;
2740        for state in [&small, &large] {
2741            Fx::target(&state.candidates[0].worktree);
2742        }
2743        let output = large.candidates[0].worktree.join("target/debug/output");
2744        std::fs::OpenOptions::new()
2745            .write(true)
2746            .open(&output)
2747            .unwrap()
2748            .set_len(1000)
2749            .unwrap();
2750        let mut disk = Fx::disk();
2751        disk.worktree_reclaim_bytes = 300;
2752        disk.min_free_bytes = u64::MAX;
2753        // Pressure must trigger reclamation even when aggregate usage is below cap.
2754        disk.worktree_limit_bytes = u64::MAX;
2755        let out = sweep(&f.home, &disk, Fx::now()).await;
2756        assert_eq!(out.freed_bytes, 1000, "{out:?}");
2757        assert!(!output.exists());
2758        assert!(
2759            small.candidates[0]
2760                .worktree
2761                .join("target/debug/output")
2762                .exists()
2763        );
2764    }
2765
2766    #[tokio::test]
2767    async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2768        let f = Fx::new().await;
2769        let first = f.run("bc00", true).await;
2770        let second = f.run("bc01", true).await;
2771        for state in [&first, &second] {
2772            Fx::target(&state.candidates[0].worktree);
2773        }
2774        // Non-declared Cargo data must not determine whether outputs fit.
2775        std::fs::create_dir_all(
2776            first.candidates[0]
2777                .worktree
2778                .join("target/debug/incremental"),
2779        )
2780        .unwrap();
2781        std::fs::write(
2782            first.candidates[0]
2783                .worktree
2784                .join("target/debug/incremental/cache"),
2785            [0; 1000],
2786        )
2787        .unwrap();
2788        let mut disk = Fx::disk();
2789        disk.worktree_reclaim_bytes = 0;
2790        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2791        disk.worktree_reclaim_bytes = 300;
2792        let out = sweep(&f.home, &disk, Fx::now()).await;
2793        assert_eq!(out.freed_bytes, 200, "{out:?}");
2794        assert!(
2795            !first.candidates[0]
2796                .worktree
2797                .join("target/debug/output")
2798                .exists()
2799        );
2800        assert!(
2801            second.candidates[0]
2802                .worktree
2803                .join("target/debug/output")
2804                .exists()
2805        );
2806        assert!(
2807            out.retained
2808                .iter()
2809                .any(|r| r.contains("declared outputs exceed"))
2810        );
2811    }
2812
2813    #[tokio::test]
2814    async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2815        let f = Fx::new().await;
2816        let mut protected = f.run("bd00", true).await;
2817        let later = f.run("bd01", true).await;
2818        for state in [&protected, &later] {
2819            Fx::target(&state.candidates[0].worktree);
2820        }
2821        let mut disk = Fx::disk();
2822        disk.worktree_reclaim_bytes = 100;
2823        disk.fold_grace_secs = 3600;
2824        let out = sweep(&f.home, &disk, Fx::now()).await;
2825        assert_eq!(out.freed_bytes, 0);
2826        assert!(
2827            out.retained
2828                .iter()
2829                .any(|r| r.contains("within grace period"))
2830        );
2831        disk.fold_grace_secs = 0;
2832        protected.driver_exited = false;
2833        protected.driver_pid = Some(std::process::id());
2834        protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2835        protected
2836            .save_under_at(&f.home, protected.updated_at)
2837            .unwrap();
2838        let out = sweep(&f.home, &disk, Fx::now()).await;
2839        assert!(
2840            out.retained
2841                .iter()
2842                .any(|r| r.contains("run live or ownership unknown"))
2843        );
2844        assert_eq!(
2845            out.freed_bytes, 200,
2846            "live target must not consume exception: {out:?}"
2847        );
2848        let output = protected.candidates[0].worktree.join("target/debug/output");
2849        assert!(output.exists());
2850        protected.driver_exited = true;
2851        protected.save_under(&f.home).unwrap();
2852        let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2853        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2854        assert!(output.exists());
2855        drop(owner);
2856        let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2857        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2858        assert!(output.exists());
2859        drop(cargo);
2860        let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2861        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2862        assert!(output.exists());
2863        drop(lease);
2864        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2865        assert!(!output.exists());
2866    }
2867
2868    #[tokio::test]
2869    async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2870        for remove_root in [true, false] {
2871            let f = Fx::new().await;
2872            let stale = Fx::new().await;
2873            let mut missing = stale.run("aa00", false).await;
2874            Fx::target(&missing.candidates[0].worktree);
2875            missing.save_under(&f.home).unwrap();
2876            if remove_root {
2877                std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2878                // An existing non-repository proves the absent root is checked
2879                // before querying Git, even when no registrations are cached.
2880                std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2881            } else {
2882                std::fs::remove_dir_all(&stale.repo).unwrap();
2883            }
2884            let healthy = f.run("aa01", false).await;
2885            Fx::target(&healthy.candidates[0].worktree);
2886
2887            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2888            assert!(report.failures.is_empty(), "{report:?}");
2889            assert_eq!(report.freed_bytes, 200, "{report:?}");
2890            assert!(
2891                !healthy.candidates[0]
2892                    .worktree
2893                    .join("target/debug/output")
2894                    .exists()
2895            );
2896            if !remove_root {
2897                // Without the repository, ownership cannot be validated.
2898                assert!(
2899                    missing.candidates[0]
2900                        .worktree
2901                        .join("target/debug/output")
2902                        .exists()
2903                );
2904            }
2905        }
2906    }
2907
2908    #[tokio::test]
2909    async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2910        let f = Fx::new().await;
2911        let mut missing = f.run("aa00", false).await;
2912        Fx::target(&missing.candidates[0].worktree);
2913        missing.repo = missing.candidates[0].worktree.clone();
2914        missing.save_under(&f.home).unwrap();
2915        std::fs::remove_dir_all(&missing.repo).unwrap();
2916        assert!(missing.worktree_root().is_dir());
2917        assert!(f.repo.is_dir());
2918
2919        let healthy = f.run("aa01", false).await;
2920        Fx::target(&healthy.candidates[0].worktree);
2921
2922        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2923        assert!(report.failures.is_empty(), "{report:?}");
2924        assert_eq!(report.freed_bytes, 200, "{report:?}");
2925        assert!(
2926            !healthy.candidates[0]
2927                .worktree
2928                .join("target/debug/output")
2929                .exists()
2930        );
2931    }
2932
2933    #[tokio::test]
2934    async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2935        let f = Fx::new().await;
2936        let broken = Fx::new().await;
2937        let mut invalid = broken.run("aa00", false).await;
2938        Fx::target(&invalid.candidates[0].worktree);
2939        invalid.save_under(&f.home).unwrap();
2940        std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2941        let healthy = f.run("aa01", false).await;
2942        Fx::target(&healthy.candidates[0].worktree);
2943
2944        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2945        assert_eq!(report.failures.len(), 1, "{report:?}");
2946        assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2947        assert_eq!(report.freed_bytes, 200, "{report:?}");
2948        assert!(
2949            !healthy.candidates[0]
2950                .worktree
2951                .join("target/debug/output")
2952                .exists()
2953        );
2954        assert!(
2955            invalid.candidates[0]
2956                .worktree
2957                .join("target/debug/output")
2958                .exists()
2959        );
2960    }
2961
2962    async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2963        let path = reconcile::run(&f.home, &task.id, &source.id, None)
2964            .await
2965            .unwrap();
2966        serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2967    }
2968
2969    #[tokio::test]
2970    async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
2971        use crate::{
2972            chat_control::{Operation, dispatch},
2973            talk::{self, Talks},
2974        };
2975        let f = Fx::new().await;
2976        let source = f.run("ae00", true).await;
2977        let clean = f.run("ae01", false).await;
2978        let queue = crate::queue::Queue::at(f.home.join("queue"));
2979        let talks = Talks::at(f.home.join("talks"));
2980        let open_talk = |id: &str| -> talk::Talk {
2981            let mut conversation = serde_json::from_value(serde_json::json!({
2982                "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
2983                "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
2984                "seat":crate::agent::SeatState::new("talk","worker",1)
2985            }))
2986            .unwrap();
2987            talks.put(&mut conversation).unwrap();
2988            conversation
2989        };
2990        let conversation = open_talk("talk-owner");
2991        let other = open_talk("talk-other");
2992        // A task this chat did not file: the owner's request covers it too.
2993        let task = f.task(&[source.clone(), clean.clone()]);
2994        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2995        let operation = |apply: Option<&str>| Operation::ReconcileRetention {
2996            task: task.id.clone(),
2997            authority: source.id.clone(),
2998            apply: apply.map(str::to_owned),
2999        };
3000        // A plan saved outside the chat is not applied through it.
3001        let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
3002            .await
3003            .unwrap();
3004        let legacy: serde_json::Value =
3005            serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
3006        let legacy_id = legacy["id"].as_str().unwrap();
3007        assert!(
3008            dispatch(
3009                &queue,
3010                &f.home,
3011                &talks,
3012                &conversation.id,
3013                operation(Some(legacy_id))
3014            )
3015            .await
3016            .is_err()
3017        );
3018        let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
3019            .await
3020            .unwrap();
3021        assert!(summary.contains("Complete review summary"));
3022        assert!(summary.contains(&source.id) && summary.contains(&clean.id));
3023        let root = f.home.join("retention-reconciliations");
3024        let displayed: serde_json::Value =
3025            serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
3026        let id = displayed["plan"].as_str().unwrap();
3027        let dir = root.join(id);
3028        let plan: serde_json::Value =
3029            serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
3030        assert_eq!(plan["version"], 2);
3031        assert!(clean.candidates[0].worktree.exists());
3032        assert!(!dir.join("approved.json").exists());
3033        // The plan stays bound to the conversation it was saved for.
3034        assert!(
3035            dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
3036                .await
3037                .is_err()
3038        );
3039        assert!(!dir.join("approved.json").exists());
3040        // No owner phrase or quote: the request itself applies the plan. An
3041        // interruption after the durable fence keeps the audit for a retry.
3042        let error = reconcile::interrupt_chat_after_fence(
3043            &f.home,
3044            &task,
3045            id,
3046            reconcile::ChatRequest {
3047                talks: &talks,
3048                conversation: &conversation.id,
3049            },
3050        )
3051        .await
3052        .unwrap_err();
3053        assert!(
3054            error.to_string().contains("simulated interruption"),
3055            "{error:#}"
3056        );
3057        assert!(clean.candidates[0].worktree.exists());
3058        assert!(dir.join("approved.json").exists());
3059        dispatch(
3060            &queue,
3061            &f.home,
3062            &talks,
3063            &conversation.id,
3064            operation(Some(id)),
3065        )
3066        .await
3067        .unwrap();
3068        assert!(!clean.candidates[0].worktree.exists());
3069        assert!(source.candidates[0].worktree.exists());
3070        let audit: serde_json::Value =
3071            serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3072        assert_eq!(audit["version"], 2);
3073        assert_eq!(
3074            audit["owner"],
3075            serde_json::json!({"conversation": conversation.id})
3076        );
3077        dispatch(
3078            &queue,
3079            &f.home,
3080            &talks,
3081            &conversation.id,
3082            operation(Some(id)),
3083        )
3084        .await
3085        .unwrap();
3086    }
3087
3088    #[tokio::test]
3089    async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3090        let f = Fx::new().await;
3091        let source = f.run("af00", true).await;
3092        let mut clean = f.run("af01", false).await;
3093        let mut dirty = f.run("af02", false).await;
3094        for state in [&mut clean, &mut dirty] {
3095            state.status = RunStatus::Prep;
3096            state.prep_complete = Some(false);
3097            state.candidates[0].empty = false;
3098            state.save_under(&f.home).unwrap();
3099        }
3100        std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3101        let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3102        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3103        let plan = reconciliation_plan(&f, &task, &source).await;
3104        let selected: Vec<_> = plan["candidates"]
3105            .as_array()
3106            .unwrap()
3107            .iter()
3108            .filter(|e| e["selected"] == true)
3109            .collect();
3110        assert_eq!(selected.len(), 1, "{plan:#}");
3111        assert_eq!(selected[0]["run"], clean.id);
3112        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3113            .await
3114            .unwrap();
3115        assert!(!clean.candidates[0].worktree.exists());
3116        assert!(dirty.candidates[0].worktree.exists());
3117        assert!(
3118            RunState::load_under(&dirty.id, &f.home)
3119                .unwrap()
3120                .retired_to
3121                .is_none()
3122        );
3123    }
3124
3125    #[tokio::test]
3126    async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3127        let f = Fx::new().await;
3128        let mut source = f.run("aa00", true).await;
3129        let alternative = f.run("aa01", true).await;
3130        let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3131            .await
3132            .unwrap();
3133        source.reviewed_commits = Some(vec![head.clone()]);
3134        source.save_under(&f.home).unwrap();
3135        let mut runs = vec![source.clone(), alternative.clone()];
3136        for i in 2..14 {
3137            runs.push(f.run(&format!("aa{i:02}"), false).await);
3138        }
3139        let mut task = f.task(&runs);
3140        task.pr_recovery = Some(crate::pr_recovery::Selection {
3141            url: "https://example.invalid/pull/1".into(),
3142            branch: "selected".into(),
3143            head: head.clone(),
3144            source_head: Some(head.clone()),
3145            source_run: Some(source.id.clone()),
3146            source_proof: None,
3147            disposition: Some(crate::pr_recovery::Disposition::Land),
3148            instruction: task.instruction.clone(),
3149            approved_at: Timestamp::now(),
3150        });
3151        source.pr_recovery = task.pr_recovery.clone();
3152        source.save_under(&f.home).unwrap();
3153        runs[0] = source.clone();
3154        crate::queue::Queue::at(f.home.join("queue"))
3155            .put(&mut task)
3156            .unwrap();
3157        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3158        let task_before = serde_json::to_value(&task).unwrap();
3159        let mut report = Report::default();
3160        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3161        assert_eq!(
3162            report.retired, 0,
3163            "automatic cleanup must still refuse blanket markers"
3164        );
3165        assert!(
3166            report
3167                .retained
3168                .iter()
3169                .any(|reason| reason.contains("explicitly retained"))
3170        );
3171        let mut total = 0;
3172        for expected in [5, 5, 2, 0] {
3173            let plan = reconciliation_plan(&f, &task, &source).await;
3174            let selected: Vec<_> = plan["candidates"]
3175                .as_array()
3176                .unwrap()
3177                .iter()
3178                .filter(|e| e["selected"] == true)
3179                .collect();
3180            assert_eq!(selected.len(), expected, "{plan:#}");
3181            for e in &selected {
3182                assert!(
3183                    RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3184                        .unwrap()
3185                        .retired_to
3186                        .is_none()
3187                );
3188            }
3189            let operation = plan["id"].as_str().unwrap();
3190            for _ in 0..2 {
3191                reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3192                    .await
3193                    .unwrap();
3194            }
3195            total += expected;
3196        }
3197        assert_eq!(total, 12);
3198        for run in &runs {
3199            let dir = f.home.join("runs").join(&run.id);
3200            assert!(dir.join("run.json").exists());
3201            assert!(dir.join("retain-recovery-sources").exists());
3202            assert!(
3203                git::branch_exists(&f.repo, &run.candidates[0].branch)
3204                    .await
3205                    .unwrap()
3206            );
3207        }
3208        assert!(source.candidates[0].worktree.exists());
3209        assert!(alternative.candidates[0].worktree.exists());
3210        assert_eq!(
3211            git::rev_parse(&f.repo, &source.candidates[0].branch)
3212                .await
3213                .unwrap(),
3214            head
3215        );
3216        assert_eq!(
3217            serde_json::to_value(
3218                crate::queue::Queue::at(f.home.join("queue"))
3219                    .get(&task.id)
3220                    .unwrap()
3221            )
3222            .unwrap(),
3223            task_before
3224        );
3225        assert_eq!(
3226            runs.iter()
3227                .filter(|r| RunState::load_under(&r.id, &f.home)
3228                    .unwrap()
3229                    .retired_to
3230                    .is_none())
3231                .count(),
3232            2
3233        );
3234    }
3235
3236    #[tokio::test]
3237    async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3238        let f = Fx::new().await;
3239        let source = f.run("ab00", true).await;
3240        let mut runs = vec![source.clone()];
3241        for i in 1..9 {
3242            runs.push(f.run(&format!("ab{i:02}"), false).await);
3243        }
3244        std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3245        std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3246        std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3247        std::fs::write(
3248            runs[3].candidates[0].worktree.join("scratch/source"),
3249            "ignored source",
3250        )
3251        .unwrap();
3252        std::fs::write(
3253            runs[4].worktree_root().join("unknown"),
3254            "unregistered source",
3255        )
3256        .unwrap();
3257        let checkpoint = git::git(
3258            &runs[5].candidates[0].worktree,
3259            &[
3260                "rev-parse",
3261                "--path-format=absolute",
3262                "--git-path",
3263                "sequencer",
3264            ],
3265        )
3266        .await
3267        .unwrap();
3268        std::fs::create_dir(checkpoint.trim()).unwrap();
3269        runs[6].driver_exited = false;
3270        runs[6].driver_pid = Some(std::process::id());
3271        runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3272        runs[6].save_under(&f.home).unwrap();
3273        runs[7].driver_exited = false;
3274        runs[7].driver_pid = None;
3275        runs[7].driver_claimed = None; // Legacy record without first-claim evidence.
3276        runs[7].save_under(&f.home).unwrap();
3277        assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3278
3279        let hidden = &runs[8].candidates[0].worktree;
3280        git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3281            .await
3282            .unwrap();
3283        std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3284        assert!(
3285            git::git(hidden, &["status", "--porcelain"])
3286                .await
3287                .unwrap()
3288                .is_empty()
3289        );
3290        let task = f.task(&runs);
3291        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3292        let plan = reconciliation_plan(&f, &task, &source).await;
3293        assert!(
3294            plan["candidates"]
3295                .as_array()
3296                .unwrap()
3297                .iter()
3298                .all(|e| e["selected"] == false),
3299            "{plan:#}"
3300        );
3301        for run in &runs {
3302            assert!(run.candidates[0].worktree.exists());
3303        }
3304    }
3305
3306    #[tokio::test]
3307    async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3308        let f = Fx::new().await;
3309        let source = f.run("ac00", true).await;
3310        let empty = f.run("ac01", false).await;
3311        let task = f.task(&[source.clone(), empty.clone()]);
3312        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3313        let plan = reconciliation_plan(&f, &task, &source).await;
3314        std::fs::write(
3315            f.home.join("runs").join(&source.id).join("new-report"),
3316            "new review checkpoint",
3317        )
3318        .unwrap();
3319        assert!(
3320            reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3321                .await
3322                .is_err()
3323        );
3324        assert!(empty.candidates[0].worktree.exists());
3325        assert!(
3326            RunState::load_under(&empty.id, &f.home)
3327                .unwrap()
3328                .retired_to
3329                .is_none()
3330        );
3331        let marker = f
3332            .home
3333            .join("runs")
3334            .join(&empty.id)
3335            .join("retain-recovery-sources");
3336        std::fs::write(&marker, "different owner").unwrap();
3337        let plan = reconciliation_plan(&f, &task, &source).await;
3338        assert!(
3339            plan["candidates"]
3340                .as_array()
3341                .unwrap()
3342                .iter()
3343                .all(|e| e["selected"] == false)
3344        );
3345    }
3346
3347    #[tokio::test]
3348    async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3349        for partial in [false, true] {
3350            let f = Fx::new().await;
3351            let source = f.run("ab10", true).await;
3352            let first = f.run("ab11", false).await;
3353            let second = f.run("ab12", false).await;
3354            let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3355            crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3356            let plan = reconciliation_plan(&f, &task, &source).await;
3357            let id = plan["id"].as_str().unwrap();
3358            let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3359            // Matching bytes without this operation's fence are not trusted.
3360            std::fs::write(&staging, b"").unwrap();
3361            assert!(
3362                reconcile::run(&f.home, &task.id, &source.id, Some(id))
3363                    .await
3364                    .is_err()
3365            );
3366            assert!(staging.exists());
3367            std::fs::remove_file(&staging).unwrap();
3368            let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3369                .await
3370                .unwrap_err();
3371            assert!(
3372                error.to_string().contains("before run state rename"),
3373                "{error:#}"
3374            );
3375            assert!(
3376                RunState::load_under(&first.id, &f.home)
3377                    .unwrap()
3378                    .retired_to
3379                    .is_some()
3380            );
3381            assert!(
3382                RunState::load_under(&second.id, &f.home)
3383                    .unwrap()
3384                    .retired_to
3385                    .is_none()
3386            );
3387            assert!(first.candidates[0].worktree.exists());
3388            assert!(second.candidates[0].worktree.exists());
3389            let interrupted = std::fs::read(&staging).unwrap();
3390            // An unrelated file must survive a refused retry unchanged.
3391            std::fs::write(&staging, b"unrelated recovery source").unwrap();
3392            assert!(
3393                reconcile::run(&f.home, &task.id, &source.id, Some(id))
3394                    .await
3395                    .is_err()
3396            );
3397            assert_eq!(
3398                std::fs::read(&staging).unwrap(),
3399                b"unrelated recovery source"
3400            );
3401            std::fs::write(&staging, interrupted).unwrap();
3402            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3403                .await
3404                .unwrap();
3405            assert!(!staging.exists());
3406            assert!(!first.candidates[0].worktree.exists());
3407            assert!(!second.candidates[0].worktree.exists());
3408            assert_eq!(
3409                RunState::load_under(&second.id, &f.home)
3410                    .unwrap()
3411                    .retired_to,
3412                Some(source.id)
3413            );
3414        }
3415    }
3416
3417    #[tokio::test]
3418    async fn reconciliation_resumes_only_its_exact_durable_fence() {
3419        let f = Fx::new().await;
3420        let source = f.run("ad00", true).await;
3421        let empty = f.run("ad01", false).await;
3422        let task = f.task(&[source.clone(), empty.clone()]);
3423        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3424        let plan = reconciliation_plan(&f, &task, &source).await;
3425        let id = plan["id"].as_str().unwrap();
3426        assert!(
3427            reconcile::interrupt_after_fence(&f.home, &task, id)
3428                .await
3429                .is_err()
3430        );
3431        assert!(empty.candidates[0].worktree.exists());
3432        assert_eq!(
3433            RunState::load_under(&empty.id, &f.home)
3434                .unwrap()
3435                .retired_to
3436                .as_ref(),
3437            Some(&source.id)
3438        );
3439        reconcile::run(&f.home, &task.id, &source.id, Some(id))
3440            .await
3441            .unwrap();
3442        assert!(!empty.candidates[0].worktree.exists());
3443        // Model an interrupted removal: restore precisely the approved base worktree.
3444        git::worktree_add_branch(
3445            &f.repo,
3446            &empty.candidates[0].worktree,
3447            "restore-empty",
3448            &f.base,
3449        )
3450        .await
3451        .unwrap();
3452        // Recreated Git administration differs from approval and is refused.
3453        assert!(
3454            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3455                .await
3456                .is_err()
3457        );
3458        assert!(empty.candidates[0].worktree.exists());
3459        let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3460        changed.prep_failure = Some("new owner edit".into());
3461        changed.save_under(&f.home).unwrap();
3462        assert!(
3463            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3464                .await
3465                .is_err()
3466        );
3467    }
3468
3469    #[tokio::test]
3470    async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3471        let f = Fx::new().await;
3472        let source = f.run("ae00", true).await;
3473        let mut prep = f.run("ae01", false).await;
3474        let candidate = prep.candidates.pop().unwrap();
3475        prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3476        prep.save_under(&f.home).unwrap();
3477        let reset = f.run("ae02", false).await;
3478        let wt = &reset.candidates[0].worktree;
3479        std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3480        git::git(wt, &["add", "."]).await.unwrap();
3481        git::git(wt, &["commit", "-m", "unique implementation"])
3482            .await
3483            .unwrap();
3484        git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3485        let unknown = f.run("ae03", false).await;
3486        let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3487        let mut raw: serde_json::Value =
3488            serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3489        raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3490        std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3491        let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3492        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3493        let plan = reconciliation_plan(&f, &task, &source).await;
3494        let selected: Vec<_> = plan["candidates"]
3495            .as_array()
3496            .unwrap()
3497            .iter()
3498            .filter(|e| e["selected"] == true)
3499            .collect();
3500        assert_eq!(selected.len(), 1, "{plan:#}");
3501        assert_eq!(selected[0]["run"], prep.id);
3502        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3503            .await
3504            .unwrap();
3505        assert!(!candidate.worktree.exists());
3506        assert!(reset.candidates[0].worktree.exists());
3507        assert!(unknown.candidates[0].worktree.exists());
3508    }
3509
3510    #[tokio::test]
3511    async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3512        let f = Fx::new().await;
3513        let mut state = f.run("aaaa", true).await;
3514        let candidate = state.candidates[0].worktree.clone();
3515        std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3516        Fx::target(&candidate);
3517        for seat in ["review-1", "review-2"] {
3518            let wt = state.worktree_root().join(seat);
3519            git::worktree_add_detached(&f.repo, &wt, &f.base)
3520                .await
3521                .unwrap();
3522            Fx::target(&wt);
3523        }
3524        state.status = RunStatus::Landing;
3525        state.save_under(&f.home).unwrap();
3526        let second = Fx::new().await;
3527        let other = second.run("bbbb", true).await;
3528        // A global store may contain repositories with different worktree roots.
3529        let mut imported = other.clone();
3530        imported.save_under(&f.home).unwrap();
3531        Fx::target(&other.candidates[0].worktree);
3532        let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3533        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3534        assert!(
3535            candidate.join("target/debug/output").exists(),
3536            "active writer protected"
3537        );
3538        assert!(
3539            !state
3540                .worktree_root()
3541                .join("review-1/target/debug/output")
3542                .exists()
3543        );
3544        assert!(
3545            !state
3546                .worktree_root()
3547                .join("review-2/target/debug/output")
3548                .exists()
3549        );
3550        assert!(
3551            !other.candidates[0]
3552                .worktree
3553                .join("target/debug/output")
3554                .exists()
3555        );
3556        assert!(out.freed_bytes >= 600);
3557        drop(guard);
3558        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3559        assert!(out.freed_bytes >= 200);
3560        assert_eq!(
3561            std::fs::read_to_string(candidate.join("source")).unwrap(),
3562            "uncommitted work"
3563        );
3564        assert!(
3565            git::branch_exists(&f.repo, &state.candidates[0].branch)
3566                .await
3567                .unwrap()
3568        );
3569    }
3570
3571    #[tokio::test]
3572    async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3573        let f = Fx::new().await;
3574        let state = f.run("cc01", false).await;
3575        let wt = &state.candidates[0].worktree;
3576        Fx::target(wt);
3577        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3578            let dir = wt.join("target").join(dir);
3579            std::fs::create_dir_all(&dir).unwrap();
3580            std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3581        }
3582        // A dep-info dependency is an input, never deletion authority.
3583        let manifest = wt.join("target/debug/output.d");
3584        let output = wt.join("target/debug/output");
3585        std::fs::write(
3586            &manifest,
3587            format!(
3588                "{}: {}\n",
3589                output.display(),
3590                wt.join("target/debug/notes.txt").display()
3591            ),
3592        )
3593        .unwrap();
3594        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3595        assert_eq!(report.freed_bytes, 200);
3596        assert!(!output.exists());
3597        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3598            assert_eq!(
3599                std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3600                "user evidence"
3601            );
3602        }
3603        // Metadata remains available for inspection and future safe rebuilds.
3604        assert!(manifest.exists());
3605        assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3606        std::fs::write(&output, "user note replacing an old binary").unwrap();
3607        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3608        assert_eq!(report.freed_bytes, 0);
3609        assert_eq!(
3610            std::fs::read_to_string(&output).unwrap(),
3611            "user note replacing an old binary"
3612        );
3613    }
3614
3615    #[tokio::test]
3616    async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3617        let f = Fx::new().await;
3618        let state = f.run("cc02", false).await;
3619        let wt = &state.candidates[0].worktree;
3620        Fx::target(wt);
3621        let mut disk = Fx::disk();
3622        disk.worktree_limit_bytes = u64::MAX;
3623        let now = Fx::now();
3624        let first = sweep(&f.home, &disk, now).await;
3625        std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3626        assert_eq!(
3627            sweep(&f.home, &disk, now).await.measured_bytes,
3628            first.measured_bytes
3629        );
3630        assert!(
3631            sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3632                .await
3633                .measured_bytes
3634                > first.measured_bytes
3635        );
3636        disk.worktree_limit_bytes = 0;
3637        assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3638    }
3639
3640    #[tokio::test]
3641    async fn unfinished_source_fold_reports_its_retention_reason() {
3642        let f = Fx::new().await;
3643        let mut source = f.run("cc03", true).await;
3644        let task = f.task(&[source.clone()]);
3645        let error = crate::graph::fold_run(&mut source, true, &f.home)
3646            .await
3647            .unwrap_err();
3648        assert!(error.to_string().contains(&task.id));
3649        assert!(source.candidates[0].worktree.exists());
3650        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3651        let error = crate::graph::fold_run(&mut source, true, &f.home)
3652            .await
3653            .unwrap_err();
3654        assert!(error.to_string().contains("explicitly retained"));
3655    }
3656
3657    #[tokio::test]
3658    async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3659        let f = Fx::new().await;
3660        let mut source = f.run("cc04", true).await;
3661        let mut task = f.task(&[source.clone()]);
3662        task.hold_manual(Some("queue settlement pending".into()));
3663        let queue = crate::queue::Queue::at(f.home.join("queue"));
3664        queue.put(&mut task).unwrap();
3665        source.merge = Some(crate::run::MergeOutcome {
3666            mode: crate::config::MergeMode::Pr,
3667            ok: true,
3668            detail: "confirmed merged".into(),
3669            empty: false,
3670        });
3671        // Publication success and terminal labels alone are insufficient.
3672        for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3673            source.status = status;
3674            assert!(
3675                crate::graph::fold_run(&mut source, true, &f.home)
3676                    .await
3677                    .is_err()
3678            );
3679            assert!(source.candidates[0].worktree.exists());
3680        }
3681        git::git(
3682            &f.repo,
3683            &["merge", "--ff-only", &source.candidates[0].branch],
3684        )
3685        .await
3686        .unwrap();
3687        git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3688            .await
3689            .unwrap();
3690        source.status = RunStatus::Merged;
3691        source.save_under(&f.home).unwrap();
3692        assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3693        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3694        assert!(
3695            crate::graph::fold_run(&mut source, true, &f.home)
3696                .await
3697                .unwrap_err()
3698                .to_string()
3699                .contains("explicitly retained")
3700        );
3701        // The fixture's explicit marker is removed only to exercise ordinary
3702        // confirmed-merge cleanup; no production cleanup removes such markers.
3703        std::fs::remove_file(
3704            f.home
3705                .join("runs")
3706                .join(&source.id)
3707                .join("retain-recovery-sources"),
3708        )
3709        .unwrap();
3710        let (folded, unreadable) = crate::clean::fold_due(
3711            &f.home.join("runs"),
3712            &f.home,
3713            &f.bay,
3714            &Fx::disk(),
3715            Fx::now(),
3716        )
3717        .await
3718        .unwrap();
3719        assert_eq!((folded, unreadable), (1, 0));
3720        assert!(!source.candidates[0].worktree.exists());
3721        let evidence = f
3722            .home
3723            .join("runs")
3724            .join(&source.id)
3725            .join("artifacts/fold-evidence");
3726        let names: Vec<_> = std::fs::read_dir(evidence)
3727            .unwrap()
3728            .map(|entry| entry.unwrap().file_name())
3729            .collect();
3730        assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3731        assert_eq!(
3732            queue.get(&task.id).unwrap().status,
3733            crate::queue::TaskStatus::Held
3734        );
3735    }
3736
3737    #[tokio::test]
3738    async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3739        for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3740            let f = Fx::new().await;
3741            let mut state = f.run("cc06", true).await;
3742            let candidate = state.candidates[0].clone();
3743            let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3744            std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3745            git::rescue_commit(&candidate.worktree, "second candidate change")
3746                .await
3747                .unwrap();
3748            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3749            state.tally = Some(crate::run::Tally {
3750                first_choice: Default::default(),
3751                borda: Default::default(),
3752                winner: 'A',
3753                rankings: 0,
3754                unanimous_initial: true,
3755                deliberated: false,
3756                changed_votes: 0,
3757                unanimous_final: true,
3758                tie_break: None,
3759                judges: 0,
3760                present: 0,
3761                quorum: 0,
3762                met_quorum: true,
3763                uncontested: Some("fixture".into()),
3764            });
3765            // A reviewer pinned to an intermediate candidate commit must also
3766            // be reclaimable, although neither commit ID lands in the squash.
3767            let reviewer = state.worktree_root().join("review-1");
3768            git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3769                .await
3770                .unwrap();
3771            // Concurrent base changes and later base advances prevent a mere
3772            // equality check against the current base tree from passing.
3773            std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3774            git::rescue_commit(&f.repo, "advance before squash")
3775                .await
3776                .unwrap();
3777            let published = if variant == "partial" {
3778                &intermediate
3779            } else {
3780                &head
3781            };
3782            git::git(&f.repo, &["merge", "--squash", published])
3783                .await
3784                .unwrap();
3785            git::git(&f.repo, &["commit", "-m", "squash candidate"])
3786                .await
3787                .unwrap();
3788            let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3789            assert!(
3790                !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3791                    .await
3792                    .unwrap()
3793                    .ok()
3794            );
3795            std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3796            git::rescue_commit(&f.repo, "advance after squash")
3797                .await
3798                .unwrap();
3799            git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3800                .await
3801                .unwrap();
3802            if matches!(variant, "checkpoint" | "reviewer") {
3803                // A reset hides unique work from HEAD, but not its reflog.
3804                std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3805                git::rescue_commit(&reviewer, "unpublished checkpoint")
3806                    .await
3807                    .unwrap();
3808                if variant == "checkpoint" {
3809                    git::git(&reviewer, &["reset", "--hard", &intermediate])
3810                        .await
3811                        .unwrap();
3812                }
3813            }
3814            if variant == "loser" {
3815                let mut loser = candidate.clone();
3816                loser.label = 'B';
3817                loser.branch = "magi/cc06/B".into();
3818                loser.worktree = state.worktree_root().join("cand-B");
3819                git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3820                    .await
3821                    .unwrap();
3822                std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3823                git::rescue_commit(&loser.worktree, "independent loser")
3824                    .await
3825                    .unwrap();
3826                state.candidates.push(loser);
3827            }
3828            state.status = RunStatus::Merged;
3829            state.merge = Some(crate::run::MergeOutcome {
3830                mode: crate::config::MergeMode::Pr,
3831                ok: true,
3832                detail: "candidate A squash merged".into(),
3833                empty: false,
3834            });
3835            state.save_under(&f.home).unwrap();
3836            let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3837            let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3838            result.unwrap();
3839            assert!(
3840                !candidate.worktree.exists() && !reviewer.exists(),
3841                "{variant}"
3842            );
3843            assert!(
3844                !git::branch_exists(&f.repo, &candidate.branch)
3845                    .await
3846                    .unwrap()
3847            );
3848            let artifacts = f
3849                .home
3850                .join("runs")
3851                .join(&state.id)
3852                .join("artifacts/fold-evidence");
3853            assert_eq!(
3854                artifacts.join(format!("{head}.patch")).exists(),
3855                variant == "partial",
3856                "only a partially published winner needs evidence: {variant}"
3857            );
3858            assert_eq!(
3859                artifacts.join(format!("{intermediate}.patch")).exists(),
3860                variant == "partial",
3861                "squash proof also covers intermediate reviewer tips: {variant}"
3862            );
3863            let patches: Vec<_> = std::fs::read_dir(&artifacts)
3864                .unwrap()
3865                .map(|entry| entry.unwrap().path())
3866                .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3867                .collect();
3868            let expected = match variant {
3869                "partial" => 2,
3870                "loser" | "reviewer" => 1,
3871                _ => 0,
3872            };
3873            assert_eq!(patches.len(), expected, "{variant}");
3874            if variant == "loser" {
3875                assert!(
3876                    std::fs::read_to_string(&patches[0])
3877                        .unwrap()
3878                        .contains("losing candidate")
3879                );
3880            }
3881            assert!(artifacts.join("manifest.json").exists());
3882            if variant == "reviewer" {
3883                assert!(
3884                    std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3885                        .unwrap()
3886                        .contains("reviewer checkpoint")
3887                );
3888            }
3889            crate::graph::fold_run(&mut state, true, &f.home)
3890                .await
3891                .unwrap();
3892        }
3893    }
3894
3895    #[tokio::test]
3896    async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3897        for missing in [true, false] {
3898            let f = Fx::new().await;
3899            let mut state = f.run("cc05", true).await;
3900            // A fetched base exists, but these local review changes were not
3901            // incorporated by ancestry, so their recovery evidence is required.
3902            git::git(
3903                &f.repo,
3904                &["update-ref", "refs/remotes/origin/main", &f.base],
3905            )
3906            .await
3907            .unwrap();
3908            let candidate = state.candidates[0].clone();
3909            if !missing {
3910                std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3911                git::rescue_commit(&f.repo, "advance base before rebase")
3912                    .await
3913                    .unwrap();
3914                git::git(&candidate.worktree, &["rebase", "main"])
3915                    .await
3916                    .unwrap();
3917                // Match the runner's post-rebase synchronization, which
3918                // clears the merge backend's transient AUTO_MERGE checkpoint.
3919                git::sync_to_head(&candidate.worktree).await.unwrap();
3920            }
3921            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3922            // Model review-only state: no implementation-time snapshot exists.
3923            state.reviewed_commits = Some(Vec::new());
3924            state.candidates[0].worktree = state.worktree_root().join("under-review");
3925            git::git(
3926                &f.repo,
3927                &[
3928                    "worktree",
3929                    "move",
3930                    candidate.worktree.to_str().unwrap(),
3931                    state.candidates[0].worktree.to_str().unwrap(),
3932                ],
3933            )
3934            .await
3935            .unwrap();
3936            let reviewer = state.worktree_root().join("review-1");
3937            git::worktree_add_detached(&f.repo, &reviewer, &head)
3938                .await
3939                .unwrap();
3940            // A reset/rebase can leave unpublished recovery history. It must
3941            // not keep any remaining owned worktree alive indefinitely.
3942            std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3943            git::rescue_commit(&reviewer, "old review history")
3944                .await
3945                .unwrap();
3946            git::git(&reviewer, &["reset", "--hard", &head])
3947                .await
3948                .unwrap();
3949            if missing {
3950                std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3951            }
3952            let run_dir = f.home.join("runs").join(&state.id);
3953            let artifacts = run_dir.join("artifacts");
3954            std::fs::create_dir_all(&artifacts).unwrap();
3955            std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3956            if missing {
3957                std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3958            }
3959            state.status = RunStatus::Merged;
3960            state.merge = Some(crate::run::MergeOutcome {
3961                mode: crate::config::MergeMode::Pr,
3962                ok: true,
3963                detail: "confirmed squash merge".into(),
3964                empty: false,
3965            });
3966            state.save_under(&f.home).unwrap();
3967            // Unexpected external paths remain protected, even if absent.
3968            let owned = state.candidates[0].worktree.clone();
3969            state.candidates[0].worktree = f.home.join("missing-external");
3970            assert!(
3971                crate::graph::fold_run(&mut state, true, &f.home)
3972                    .await
3973                    .unwrap_err()
3974                    .to_string()
3975                    .contains("outside the inventoried")
3976            );
3977            state.candidates[0].worktree = owned;
3978            // A live output borrower blocks the entire preflight.
3979            let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
3980            assert!(
3981                crate::graph::fold_run(&mut state, true, &f.home)
3982                    .await
3983                    .is_err()
3984            );
3985            assert!(reviewer.exists());
3986            drop(guard);
3987            // Failure to persist evidence must not cause partial removal.
3988            let evidence = artifacts.join("fold-evidence");
3989            std::fs::write(&evidence, "blocked").unwrap();
3990            assert!(
3991                crate::graph::fold_run(&mut state, true, &f.home)
3992                    .await
3993                    .is_err()
3994            );
3995            assert!(reviewer.exists());
3996            std::fs::remove_file(&evidence).unwrap();
3997            std::fs::create_dir_all(&evidence).unwrap();
3998            let oversized = evidence.join("capacity-fixture");
3999            std::fs::File::create(&oversized)
4000                .unwrap()
4001                .set_len(64 * 1024 * 1024)
4002                .unwrap();
4003            assert!(
4004                crate::graph::fold_run(&mut state, true, &f.home)
4005                    .await
4006                    .is_err()
4007            );
4008            assert!(reviewer.exists());
4009            std::fs::remove_file(oversized).unwrap();
4010            // Ignored recovery files and unfinished Git operations still hold
4011            // all sources, even for a confirmed merge.
4012            std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
4013            std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
4014            assert!(
4015                crate::graph::fold_run(&mut state, true, &f.home)
4016                    .await
4017                    .is_err()
4018            );
4019            std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
4020            let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
4021                .await
4022                .unwrap();
4023            std::fs::write(&checkpoint, &head).unwrap();
4024            assert!(
4025                crate::graph::fold_run(&mut state, true, &f.home)
4026                    .await
4027                    .is_err()
4028            );
4029            std::fs::remove_file(checkpoint).unwrap();
4030            let (folded, unreadable) = crate::clean::fold_due(
4031                &f.home.join("runs"),
4032                &f.home,
4033                &f.bay,
4034                &Fx::disk(),
4035                Fx::now(),
4036            )
4037            .await
4038            .unwrap();
4039            if folded != 1 {
4040                panic!(
4041                    "normal cleanup failed: {:#}",
4042                    crate::graph::fold_run(&mut state, true, &f.home)
4043                        .await
4044                        .unwrap_err()
4045                );
4046            }
4047            assert_eq!((folded, unreadable), (1, 0));
4048            assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4049            state = RunState::load_under(&state.id, &f.home).unwrap();
4050            assert!(state.candidates[0].folded);
4051            let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4052            crate::graph::fold_run(&mut state, true, &f.home)
4053                .await
4054                .unwrap();
4055            assert_eq!(
4056                std::fs::read(evidence.join("manifest.json")).unwrap(),
4057                manifest
4058            );
4059            git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4060                .await
4061                .unwrap();
4062            git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4063            assert!(
4064                !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4065                    .await
4066                    .unwrap()
4067                    .ok()
4068            );
4069            // Restore the modified file into a fresh repository from artifact
4070            // bytes alone; no original commit or blob is consulted.
4071            let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4072            let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4073                .as_str()
4074                .unwrap();
4075            let recovered = f.home.join("recovered");
4076            std::fs::create_dir_all(&recovered).unwrap();
4077            git::git(&recovered, &["init"]).await.unwrap();
4078            std::fs::write(
4079                recovered.join("source"),
4080                std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4081            )
4082            .unwrap();
4083            git::git(
4084                &recovered,
4085                &[
4086                    "apply",
4087                    evidence.join(format!("{head}.patch")).to_str().unwrap(),
4088                ],
4089            )
4090            .await
4091            .unwrap();
4092            assert_eq!(
4093                std::fs::read_to_string(recovered.join("source")).unwrap(),
4094                "cc05"
4095            );
4096            assert!(
4097                !std::fs::read(evidence.join(format!("{head}.patch")))
4098                    .unwrap()
4099                    .is_empty()
4100            );
4101            assert_eq!(
4102                std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4103                "review findings"
4104            );
4105            if missing {
4106                assert_eq!(
4107                    std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4108                    "original snapshot"
4109                );
4110            } else {
4111                assert!(!artifacts.join("cand-A.patch").exists());
4112            }
4113            assert!(run_dir.join("run.json").exists());
4114        }
4115    }
4116
4117    #[tokio::test]
4118    async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4119        use std::os::unix::fs::PermissionsExt as _;
4120        let f = Fx::new().await;
4121        let source = f.run("dd03", true).await;
4122        let empty = f.run("dd04", false).await;
4123        let task = f.task(&[source, empty.clone()]);
4124        let entered = f.repo.join("proof-entered");
4125        let release = f.repo.join("proof-release");
4126        let hook = f.repo.join("proof-monitor");
4127        std::fs::write(&hook, format!(
4128            "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4129            entered.display(), release.display())).unwrap();
4130        std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4131        git::git(
4132            &f.repo,
4133            &["config", "core.fsmonitor", hook.to_str().unwrap()],
4134        )
4135        .await
4136        .unwrap();
4137        struct ReleaseProof(PathBuf);
4138        impl Drop for ReleaseProof {
4139            fn drop(&mut self) {
4140                let _ = std::fs::write(&self.0, "continue");
4141            }
4142        }
4143        let _release_on_panic = ReleaseProof(release.clone());
4144        let home = f.home.clone();
4145        let sweep = tokio::spawn(async move {
4146            let mut report = Report::default();
4147            retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4148            report
4149        });
4150        // These are deadlock watchdogs, not speed assertions. The proof stays
4151        // blocked until the task update completes, even on a loaded runner.
4152        let watchdog = std::time::Duration::from_secs(60);
4153        tokio::time::timeout(watchdog, async {
4154            while !entered.exists() {
4155                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4156            }
4157        })
4158        .await
4159        .expect("Git retirement proof did not enter the blocking hook");
4160        let queue = crate::queue::Queue::at(f.home.join("queue"));
4161        let mut selected = queue.get(&task.id).unwrap();
4162        selected.resume_override = Some(crate::queue::OperatorResume {
4163            question_id: "during-slow-proof".into(),
4164            at: Fx::now(),
4165            conductor_rehold: None,
4166            forced: false,
4167            pinned_run: Some(empty.id.clone()),
4168        });
4169        let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4170        // The proof is released only after the update returns, so correctness
4171        // is decided by ordering, not speed. The 60-second bound only detects
4172        // a writer blocked behind the proof; it is not a performance threshold.
4173        let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4174            Ok(result) => {
4175                result.unwrap().unwrap();
4176                true
4177            }
4178            Err(_) => {
4179                std::fs::write(&release, "continue").unwrap();
4180                update.await.unwrap().unwrap();
4181                false
4182            }
4183        };
4184        std::fs::write(release, "continue").unwrap();
4185        let report = tokio::time::timeout(watchdog, sweep)
4186            .await
4187            .expect("Git retirement proof did not finish after release")
4188            .unwrap();
4189        assert!(
4190            updated_while_blocked,
4191            "task writers must not wait behind Git retirement proof"
4192        );
4193        assert_eq!(report.retired, 0);
4194        assert!(report.retained.iter().any(|r| r.contains("task changed")));
4195        assert!(empty.candidates[0].worktree.exists());
4196        assert!(
4197            RunState::load_under(&empty.id, &f.home)
4198                .unwrap()
4199                .retired_to
4200                .is_none()
4201        );
4202    }
4203
4204    #[tokio::test]
4205    async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4206        let f = Fx::new().await;
4207        let source = f.run("dd01", true).await;
4208        let mut empty = f.run("dd02", false).await;
4209        let expected = f.task(&[source.clone(), empty.clone()]);
4210        let queue = crate::queue::Queue::at(f.home.join("queue"));
4211        let _claim = queue.claim(&expected.id).unwrap();
4212        // The proof holds run/output leases, never the task write lock.
4213        let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4214        let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4215        let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4216        let mut selected = queue.get(&expected.id).unwrap();
4217        selected.resume_override = Some(crate::queue::OperatorResume {
4218            question_id: "new-owner-decision".into(),
4219            at: Fx::now(),
4220            conductor_rehold: None,
4221            forced: false,
4222            pinned_run: Some(empty.id.clone()),
4223        });
4224        queue.put(&mut selected).unwrap();
4225        let error = commit_retirement(
4226            &f.home,
4227            &queue,
4228            &expected,
4229            &mut empty,
4230            &source.id,
4231            &expected.runs,
4232            Fx::now(),
4233        )
4234        .unwrap_err();
4235        assert!(error.to_string().contains("task changed"));
4236        assert!(
4237            RunState::load_under(&empty.id, &f.home)
4238                .unwrap()
4239                .retired_to
4240                .is_none()
4241        );
4242        assert!(empty.candidates[0].worktree.exists());
4243        assert_eq!(
4244            queue
4245                .get(&expected.id)
4246                .unwrap()
4247                .resume_override
4248                .unwrap()
4249                .pinned_run,
4250            Some(empty.id.clone())
4251        );
4252        assert!(
4253            !f.home
4254                .join("recovery-lineages")
4255                .join(format!("{}.json", expected.id))
4256                .exists()
4257        );
4258        // Re-reading the new snapshot must not bypass its explicit pin either.
4259        assert!(
4260            commit_retirement(
4261                &f.home,
4262                &queue,
4263                &selected,
4264                &mut empty,
4265                &source.id,
4266                &selected.runs,
4267                Fx::now()
4268            )
4269            .unwrap_err()
4270            .to_string()
4271            .contains("selected this recovery source")
4272        );
4273    }
4274
4275    #[tokio::test]
4276    async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4277        let f = Fx::new().await;
4278        let state = f.run("cccc", false).await;
4279        let wt = &state.candidates[0].worktree;
4280        Fx::target(wt);
4281        std::fs::write(wt.join("target/notes"), "user file").unwrap();
4282        std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4283        assert!(
4284            !declared_outputs(&wt.join("target"))
4285                .unwrap()
4286                .contains(&wt.join("target/debug/link"))
4287        );
4288        std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4289        let cargo = cargo_locks(&wt.join("target")).unwrap();
4290        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4291        assert_eq!(out.freed_bytes, 0);
4292        assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4293        drop(cargo);
4294        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4295        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4296        assert_eq!(out.freed_bytes, 0);
4297        drop(driver);
4298        let exclusive = OutputLease::take(wt, true).unwrap();
4299        assert!(
4300            OutputLease::borrow(wt).is_err(),
4301            "resume cannot start a writer during cleanup"
4302        );
4303        drop(exclusive);
4304        assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4305        assert_eq!(
4306            std::fs::read(wt.join("target/notes")).unwrap(),
4307            b"user file"
4308        );
4309    }
4310
4311    #[tokio::test]
4312    async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4313        for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4314            let f = Fx::new().await;
4315            let source = f.run("aa10", true).await;
4316            let empty = f.run("aa11", false).await;
4317            f.task(&[source, empty.clone()]);
4318            let wt = &empty.candidates[0].worktree;
4319            Fx::target(wt);
4320            let path = wt.join("target").join(leftover);
4321            if leftover.ends_with(".log") {
4322                std::fs::write(&path, "keep log\n").unwrap();
4323            } else {
4324                // Empty unknown directories are not present in residue_files.
4325                std::fs::create_dir(&path).unwrap();
4326            }
4327            let mut disk = Fx::disk();
4328            disk.auto_fold = true;
4329            let report = sweep(&f.home, &disk, Fx::now()).await;
4330            assert_eq!(report.freed_bytes, 200, "{report:?}");
4331            assert_eq!(report.retired, 0, "{report:?}");
4332            assert!(!wt.join("target/debug/output").exists());
4333            assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4334            assert!(path.exists());
4335            if leftover.ends_with(".log") {
4336                assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4337            }
4338            assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4339            assert!(worktrees(&empty).await.unwrap().contains(wt));
4340            assert!(
4341                git::branch_exists(&f.repo, &empty.candidates[0].branch)
4342                    .await
4343                    .unwrap()
4344            );
4345            assert!(
4346                RunState::load_under(&empty.id, &f.home)
4347                    .unwrap()
4348                    .retired_to
4349                    .is_none()
4350            );
4351        }
4352    }
4353
4354    #[tokio::test]
4355    async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4356        let f = Fx::new().await;
4357        let state = f.run("aa12", false).await;
4358        let wt = &state.candidates[0].worktree;
4359        Fx::target(wt);
4360        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4361        assert_eq!(report.freed_bytes, 200, "{report:?}");
4362        assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4363        for name in ["tmp", "magi-tools", "log"] {
4364            let path = wt.join("target").join(name);
4365            std::fs::create_dir(&path).unwrap();
4366            assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4367            std::fs::remove_dir(path).unwrap();
4368        }
4369        assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4370    }
4371
4372    #[tokio::test]
4373    async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4374        for invalid in [
4375            "tag",
4376            "identity",
4377            "tracked",
4378            "profile-file",
4379            "profile-symlink",
4380            "identity-symlink",
4381        ] {
4382            let f = Fx::new().await;
4383            let state = f.run("aa13", false).await;
4384            let wt = &state.candidates[0].worktree;
4385            Fx::target(wt);
4386            std::fs::create_dir(wt.join("target/tmp")).unwrap();
4387            let target = wt.join("target");
4388            match invalid {
4389                "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4390                "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4391                "tracked" => {
4392                    git::git(wt, &["add", "-f", "target/debug/output"])
4393                        .await
4394                        .unwrap();
4395                }
4396                "profile-file" => {
4397                    std::fs::write(target.join("release"), "keep").unwrap();
4398                }
4399                "profile-symlink" => {
4400                    std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4401                        .unwrap();
4402                }
4403                "identity-symlink" => {
4404                    std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4405                        .unwrap();
4406                    std::os::unix::fs::symlink(
4407                        target.join("saved-info"),
4408                        target.join(".rustc_info.json"),
4409                    )
4410                    .unwrap();
4411                }
4412                _ => unreachable!(),
4413            }
4414            let before = std::fs::read(target.join("debug/output")).unwrap();
4415            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4416            assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4417            assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4418            assert!(target.join("tmp").is_dir());
4419        }
4420    }
4421
4422    #[tokio::test]
4423    async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4424        let f = Fx::new().await;
4425        let source = f.run("0000", true).await;
4426        let empty = f.run("0001", false).await;
4427        let changed = f.run("0002", false).await;
4428        let unknown = f.run("0003", false).await;
4429        let unproven = f.run("0004", false).await;
4430        f.task(&[
4431            source.clone(),
4432            empty.clone(),
4433            changed.clone(),
4434            unknown.clone(),
4435            unproven.clone(),
4436        ]);
4437        for run in [&empty, &changed, &unknown] {
4438            Fx::target(&run.candidates[0].worktree);
4439        }
4440        let reviewer = empty.worktree_root().join("review-1");
4441        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4442            .await
4443            .unwrap();
4444        Fx::target(&reviewer);
4445        let mut disk = Fx::disk();
4446        disk.auto_fold = false;
4447        let report = sweep(&f.home, &disk, Fx::now()).await;
4448        assert_eq!(report.freed_bytes, 800, "{report:?}");
4449        let wt = &empty.candidates[0].worktree;
4450        assert!(wt.join("target/debug/output.d").exists());
4451        assert!(!wt.join("target/debug/output").exists());
4452        let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4453        let proof: CargoResidue =
4454            serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4455        assert_eq!(
4456            proof.files[Path::new("debug/output.d")],
4457            std::fs::read(wt.join("target/debug/output.d")).unwrap()
4458        );
4459        std::fs::write(
4460            changed.candidates[0].worktree.join("target/debug/output.d"),
4461            "owner notes",
4462        )
4463        .unwrap();
4464        std::fs::write(
4465            unknown.candidates[0].worktree.join("target/debug/notes"),
4466            "keep",
4467        )
4468        .unwrap();
4469        Fx::target(&unproven.candidates[0].worktree);
4470        std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4471        disk.auto_fold = true;
4472        let locks = cargo_locks(&wt.join("target")).unwrap();
4473        let mut report = Report::default();
4474        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4475        assert_eq!(report.retired, 0, "{report:?}");
4476        assert!(
4477            report
4478                .retained
4479                .iter()
4480                .any(|reason| reason.contains("Cargo build"))
4481        );
4482        drop(locks);
4483        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4484        assert_eq!(report.retired, 1, "{report:?}");
4485        assert!(!wt.exists());
4486        assert!(!reviewer.exists());
4487        assert!(
4488            residue_proof(&f.home, &empty.id, &reviewer)
4489                .unwrap()
4490                .exists()
4491        );
4492        assert!(
4493            proof_path.exists(),
4494            "metadata evidence outlives the worktree"
4495        );
4496        assert_eq!(
4497            RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4498            Some(source.id)
4499        );
4500        for run in [&changed, &unknown, &unproven] {
4501            assert!(run.candidates[0].worktree.exists());
4502            assert!(
4503                RunState::load_under(&run.id, &f.home)
4504                    .unwrap()
4505                    .retired_to
4506                    .is_none()
4507            );
4508        }
4509    }
4510
4511    #[tokio::test]
4512    async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4513        let f = Fx::new().await;
4514        let source = f.run("0000", true).await;
4515        let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4516            .await
4517            .unwrap();
4518        let mut runs = vec![source.clone()];
4519        for n in 1..=12 {
4520            runs.push(f.run(&format!("{n:04}"), false).await);
4521        }
4522        let dirty = runs[1].candidates[0].worktree.clone();
4523        std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4524        let ignored = runs[2].candidates[0].worktree.clone();
4525        std::fs::create_dir(ignored.join("scratch")).unwrap();
4526        std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4527        // A missing candidate array does not prove that an unrecorded worktree
4528        // is empty: discovery must inspect Git and the worktree itself.
4529        runs[1].candidates.clear();
4530        runs[1].save_under(&f.home).unwrap();
4531        let task = f.task(&runs);
4532        let mut total = 0;
4533        for _ in 0..4 {
4534            let mut report = Report::default();
4535            retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4536            assert!(report.retired <= 5);
4537            total += report.retired;
4538        }
4539        assert_eq!(total, 10);
4540        assert!(dirty.exists());
4541        assert!(ignored.exists());
4542        assert_eq!(
4543            git::rev_parse(&f.repo, &source.candidates[0].branch)
4544                .await
4545                .unwrap(),
4546            source_head
4547        );
4548        let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4549        assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4550        assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4551        let mut released = task.clone();
4552        released.release();
4553        assert_eq!(
4554            crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4555                RunState::load_under(id, &f.home)
4556            }),
4557            Some(source.id.clone())
4558        );
4559
4560        assert!(
4561            f.home
4562                .join("runs")
4563                .join(&retired.id)
4564                .join("run.json")
4565                .exists()
4566        );
4567        assert!(
4568            f.home
4569                .join("recovery-lineages")
4570                .join(format!("{}.json", task.id))
4571                .exists()
4572        );
4573    }
4574
4575    #[tokio::test]
4576    async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4577        let f = Fx::new().await;
4578        let a = f.run("dddd", true).await;
4579        let b = f.run("eeee", true).await;
4580        let empty = f.run("ffff", false).await;
4581        let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4582        assert!(
4583            crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4584            "unfinished queue history protects older source branches from ordinary folding"
4585        );
4586        let mut prior = a.clone();
4587        assert!(
4588            crate::graph::fold_run(&mut prior, true, &f.home)
4589                .await
4590                .unwrap_err()
4591                .to_string()
4592                .contains(&task.id)
4593        );
4594        let mut report = Report::default();
4595        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4596        assert_eq!(report.retired, 0);
4597        assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4598        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4599        Fx::target(&a.candidates[0].worktree);
4600        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4601        assert!(
4602            report.freed_bytes > 0,
4603            "source retention does not retain regenerable outputs"
4604        );
4605        assert!(a.candidates[0].worktree.exists());
4606        assert!(b.candidates[0].worktree.exists());
4607        assert!(empty.candidates[0].worktree.exists());
4608    }
4609
4610    #[tokio::test]
4611    async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4612        let f = Fx::new().await;
4613        let authority = f.run("p000", true).await;
4614        let question_run = f.run("p001", false).await;
4615        let mut checkpoint = f.run("p002", false).await;
4616        checkpoint.land_approval = Some(crate::run::LandApproval {
4617            question: "pending".into(),
4618            head: f.base.clone(),
4619        });
4620        checkpoint.save_under(&f.home).unwrap();
4621        let retained = f.run("p003", false).await;
4622        std::fs::write(
4623            f.home
4624                .join("runs")
4625                .join(&retained.id)
4626                .join("retain-recovery-sources"),
4627            "owner",
4628        )
4629        .unwrap();
4630        let mut partial = f.run("p004", false).await;
4631        let reviewer = partial.worktree_root().join("review-1");
4632        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4633            .await
4634            .unwrap();
4635        // Model a crash after the retirement fence but only one removal.
4636        partial.retired_to = Some(authority.id.clone());
4637        partial.save_under(&f.home).unwrap();
4638        assert!(
4639            git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4640                .await
4641                .unwrap()
4642        );
4643        let git_checkpoint = f.run("p005", false).await;
4644        let location = git::git(
4645            &git_checkpoint.candidates[0].worktree,
4646            &[
4647                "rev-parse",
4648                "--path-format=absolute",
4649                "--git-path",
4650                "sequencer",
4651            ],
4652        )
4653        .await
4654        .unwrap();
4655        std::fs::create_dir(location.trim()).unwrap();
4656        let mut preservation = f.run("p006", false).await;
4657        preservation
4658            .candidate_preservation
4659            .insert('A', "pending preservation".into());
4660        preservation.save_under(&f.home).unwrap();
4661        let task = f.task(&[
4662            preservation.clone(),
4663            git_checkpoint.clone(),
4664            authority.clone(),
4665            question_run.clone(),
4666            checkpoint.clone(),
4667            retained.clone(),
4668            partial.clone(),
4669        ]);
4670        let questions = crate::ask::Questions::at(f.home.join("questions"));
4671        let mut question = crate::ask::Question::new(
4672            question_run.id.clone(),
4673            "implement".into(),
4674            "fixture".into(),
4675            "owner decision".into(),
4676            String::new(),
4677            vec!["keep".into()],
4678        );
4679        questions.put(&mut question).unwrap();
4680        let mut report = Report::default();
4681        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4682        assert_eq!(report.retired, 1, "{report:?}");
4683        assert!(!reviewer.exists());
4684        for state in [
4685            &question_run,
4686            &checkpoint,
4687            &retained,
4688            &git_checkpoint,
4689            &preservation,
4690        ] {
4691            assert!(state.candidates[0].worktree.exists());
4692            assert!(
4693                RunState::load_under(&state.id, &f.home)
4694                    .unwrap()
4695                    .retired_to
4696                    .is_none()
4697            );
4698        }
4699        assert!(questions.get(&question.id).unwrap().status.open());
4700        assert_eq!(
4701            crate::queue::Queue::at(f.home.join("queue"))
4702                .get(&task.id)
4703                .unwrap()
4704                .status,
4705            task.status
4706        );
4707    }
4708
4709    #[tokio::test]
4710    async fn preservation_schema_migration_remains_distinct_from_retirement() {
4711        let f = Fx::new().await;
4712        let mut state = f.run("p007", false).await;
4713        state.schema = 15;
4714        state.save_under(&f.home).unwrap();
4715        let native = RunState::load_under(&state.id, &f.home).unwrap();
4716        assert_eq!(native.schema, crate::run::SCHEMA);
4717        assert!(native.candidate_preservation.is_empty());
4718        assert!(native.retired_to.is_none());
4719        state.schema = 14;
4720        state.save_under(&f.home).unwrap();
4721        let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4722        assert!(legacy.candidate_preservation.contains_key(&'A'));
4723        legacy.save_under(&f.home).unwrap();
4724        let restored = RunState::load_under(&state.id, &f.home).unwrap();
4725        assert_eq!(
4726            restored.candidate_preservation,
4727            legacy.candidate_preservation
4728        );
4729    }
4730
4731    #[tokio::test]
4732    async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4733        let f = Fx::new().await;
4734        let cache = f.repo.join("cache");
4735        let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4736        std::fs::create_dir_all(&cache).unwrap();
4737        assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4738        assert!(
4739            crate::cache::maintenance_prune(&f.home, &cache, 1)
4740                .unwrap()
4741                .is_none()
4742        );
4743        drop(guard);
4744        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4745    }
4746
4747    #[tokio::test]
4748    async fn launcher_death_does_not_release_live_tool_output_reservations() {
4749        let f = Fx::new().await;
4750        let run = f.run("abcd", false).await;
4751        let wt = &run.candidates[0].worktree;
4752        let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4753        helper
4754            .quiet()
4755            .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4756            .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4757            .env("MAGI_TEST_WORKTREE", wt)
4758            .stdin(std::process::Stdio::null())
4759            .stdout(std::process::Stdio::null())
4760            .stderr(std::process::Stdio::null())
4761            .kill_on_drop(true);
4762        assert!(
4763            tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4764                .await
4765                .unwrap()
4766                .unwrap()
4767                .success()
4768        );
4769        let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4770            .unwrap()
4771            .trim()
4772            .parse()
4773            .unwrap();
4774        struct KillTool(i32);
4775        impl Drop for KillTool {
4776            fn drop(&mut self) {
4777                // SAFETY: PID belongs to the bounded helper above.
4778                unsafe { libc::kill(self.0, libc::SIGKILL) };
4779            }
4780        }
4781        let tool = KillTool(pid);
4782        assert!(OutputLease::take(wt, true).is_err());
4783        assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4784        drop(tool);
4785        tokio::time::timeout(std::time::Duration::from_secs(5), async {
4786            loop {
4787                if OutputLease::take(wt, true).is_ok()
4788                    && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4789                {
4790                    break;
4791                }
4792                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4793            }
4794        })
4795        .await
4796        .unwrap();
4797    }
4798
4799    #[tokio::test]
4800    async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4801        let f = Fx::new().await;
4802        let run = f.run("abcd", false).await;
4803        let wt = &run.candidates[0].worktree;
4804        let cache = f.repo.join("cache");
4805        std::fs::create_dir(&cache).unwrap();
4806        for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4807            let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4808            let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4809            cmd.quiet()
4810                .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4811                .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4812                .stdin(std::process::Stdio::null())
4813                .stdout(std::process::Stdio::null())
4814                .stderr(std::process::Stdio::null())
4815                .kill_on_drop(true);
4816            if clear_owner {
4817                cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4818            } else {
4819                cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4820            }
4821            let gib = 1024 * 1024 * 1024;
4822            let mut leases = vec![
4823                OutputLease::borrow(wt).unwrap().unwrap(),
4824                OutputLease::cache(&f.home, &cache, false).unwrap(),
4825            ];
4826            leases.extend(
4827                OutputLease::admit_with(
4828                    &f.home,
4829                    std::slice::from_ref(wt),
4830                    8 * gib,
4831                    |_| Ok(9 * gib),
4832                )
4833                .unwrap(),
4834            );
4835            let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4836            assert!(
4837                tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4838                    .await
4839                    .unwrap()
4840                    .unwrap()
4841                    .success()
4842            );
4843            let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4844            struct KillTool(Option<i32>);
4845            impl Drop for KillTool {
4846                fn drop(&mut self) {
4847                    // SAFETY: PID from our helper; kill also bounds panic paths.
4848                    if let Some(pid) = self.0 {
4849                        unsafe { libc::kill(pid, libc::SIGKILL) };
4850                    }
4851                }
4852            }
4853            let mut tool = KillTool(Some(pid));
4854            // The CLI has already exited and cannot supply a parent-tree
4855            // relationship. Either the witness or inherited environment must
4856            // retain ownership; test each independently.
4857            assert!(OutputLease::take(wt, true).is_err());
4858            let finish = if cancel {
4859                drop(running);
4860                None
4861            } else {
4862                Some(tokio::spawn(async move { running.finish().await }))
4863            };
4864            if clear_owner {
4865                tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4866                assert!(crate::proc::pid_alive(pid as u32));
4867                assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4868                assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4869                assert!(
4870                    OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4871                        9 * gib
4872                    ))
4873                    .is_err(),
4874                    "orphan still owns its disk reservation"
4875                );
4876                assert!(
4877                    crate::cache::maintenance_prune(&f.home, &cache, 0)
4878                        .unwrap()
4879                        .is_none()
4880                );
4881                if let Some(finish) = &finish {
4882                    assert!(!finish.is_finished());
4883                }
4884                // Witness-only ownership waits for natural/external tool exit.
4885                unsafe { libc::kill(pid, libc::SIGKILL) };
4886            }
4887            if let Some(finish) = finish {
4888                tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4889                    .await
4890                    .unwrap()
4891                    .unwrap();
4892            }
4893            tokio::time::timeout(std::time::Duration::from_secs(5), async {
4894                loop {
4895                    if OutputLease::take(wt, true).is_ok()
4896                        && OutputLease::cache(&f.home, &cache, true).is_ok()
4897                        && OutputLease::admit_with(
4898                            &f.home,
4899                            std::slice::from_ref(wt),
4900                            8 * gib,
4901                            |_| Ok(9 * gib),
4902                        )
4903                        .is_ok()
4904                    {
4905                        // Token-owned orphans must be stopped by cleanup itself.
4906                        if !clear_owner {
4907                            assert!(!crate::proc::pid_alive(pid as u32));
4908                        }
4909                        break;
4910                    }
4911                    tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4912                }
4913            })
4914            .await
4915            .unwrap();
4916            tool.0 = None;
4917        }
4918    }
4919
4920    #[tokio::test]
4921    async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4922        let f = Fx::new().await;
4923        let run = f.run("abcd", false).await;
4924        let wt = &run.candidates[0].worktree;
4925        let cache = f.repo.join("cache");
4926        std::fs::create_dir(&cache).unwrap();
4927        let pid_file = f.repo.join("child-pid");
4928        let mut cmd = tokio::process::Command::new("sh");
4929        cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4930            .quiet()
4931            .env("CHILD_PID_FILE", &pid_file)
4932            .process_group(0)
4933            .kill_on_drop(true);
4934        let running = crate::seat_process::Running::spawn(
4935            cmd,
4936            None,
4937            vec![
4938                OutputLease::borrow(wt).unwrap().unwrap(),
4939                OutputLease::cache(&f.home, &cache, false).unwrap(),
4940            ],
4941        )
4942        .unwrap();
4943        tokio::time::timeout(std::time::Duration::from_secs(5), async {
4944            while !pid_file.exists() {
4945                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4946            }
4947        })
4948        .await
4949        .unwrap();
4950        assert!(OutputLease::take(wt, true).is_err());
4951        assert!(
4952            crate::cache::maintenance_prune(&f.home, &cache, 0)
4953                .unwrap()
4954                .is_none()
4955        );
4956        let pid: u32 = std::fs::read_to_string(pid_file)
4957            .unwrap()
4958            .trim()
4959            .parse()
4960            .unwrap();
4961        drop(running);
4962        tokio::time::timeout(std::time::Duration::from_secs(10), async {
4963            loop {
4964                if let Ok(guard) = OutputLease::take(wt, true) {
4965                    assert!(
4966                        !crate::proc::pid_alive(pid),
4967                        "output lock released before tool death"
4968                    );
4969                    drop(guard);
4970                    break;
4971                }
4972                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4973            }
4974        })
4975        .await
4976        .unwrap();
4977        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4978    }
4979
4980    #[tokio::test]
4981    async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
4982        let f = Fx::new().await;
4983        let reason = tokio::time::timeout(
4984            std::time::Duration::from_secs(2),
4985            emergency(vec![f.repo.clone()], u64::MAX),
4986        )
4987        .await
4988        .unwrap();
4989        assert!(reason.contains("disk pressure"));
4990        assert!(
4991            tokio::time::timeout(
4992                std::time::Duration::from_millis(10),
4993                emergency(vec![f.repo.clone()], 0)
4994            )
4995            .await
4996            .is_err()
4997        );
4998        assert_eq!(
4999            std::fs::read_to_string(f.repo.join("source")).unwrap(),
5000            "base\n"
5001        );
5002    }
5003
5004    #[tokio::test]
5005    async fn automatic_requeues_cannot_refund_failed_input_generation() {
5006        let f = Fx::new().await;
5007        let mut run = f.run("1234", false).await;
5008        run.prep_failure = Some("cherry-pick conflict".into());
5009        let mut task = f.task(std::slice::from_ref(&run));
5010        let inputs = crate::queue::recovery_inputs(&run.sources);
5011        for _ in 0..108 {
5012            task.recovery_failed(&run);
5013            task.requeue();
5014            assert_eq!(task.attempts, 0);
5015        }
5016        let bytes = serde_json::to_vec(&task).unwrap();
5017        let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
5018        assert!(
5019            restored
5020                .recovery_exhausted(&run.base_commit, &inputs, 2)
5021                .is_some()
5022        );
5023        assert!(
5024            restored
5025                .recovery_exhausted("different-base", &inputs, 2)
5026                .is_none()
5027        );
5028        let original_base = run.base_commit.clone();
5029        run.base_commit = "different-base".into();
5030        restored.recovery_failed(&run);
5031        assert!(
5032            restored
5033                .recovery_exhausted(&original_base, &inputs, 2)
5034                .is_some(),
5035            "returning to old inputs does not replenish their budget"
5036        );
5037        assert!(
5038            restored
5039                .recovery_exhausted(&run.base_commit, &inputs, 2)
5040                .is_none()
5041        );
5042        restored.release_operator("test-owner", None);
5043        assert!(
5044            restored
5045                .recovery_exhausted(&run.base_commit, &inputs, 2)
5046                .is_none()
5047        );
5048    }
5049    async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5050        let mut state = f.run(tag, useful).await;
5051        state.status = RunStatus::Superseded;
5052        state.save_under(&f.home).unwrap();
5053        state
5054    }
5055
5056    fn historical_review(state: &mut RunState, head: &str) {
5057        state.reviews.push(
5058            serde_json::from_value(serde_json::json!({
5059                "round": state.reviews.len() + 1, "head": head, "reviews": []
5060            }))
5061            .unwrap(),
5062        );
5063    }
5064
5065    async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5066        let path = state.worktree_root().join("review-1");
5067        git::worktree_add_detached(&f.repo, &path, head)
5068            .await
5069            .unwrap();
5070        path
5071    }
5072
5073    #[tokio::test]
5074    async fn source_eviction_separates_run_authority_from_read_seats() {
5075        for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5076            let f = Fx::new().await;
5077            let mut state = inactive_source(&f, "ef01", true).await;
5078            state.status = status;
5079            state.pr = Some(
5080                serde_json::from_value(serde_json::json!({
5081                    "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5082                    "checks":"green", "round":1, "rounds":1
5083                }))
5084                .unwrap(),
5085            );
5086            state.pr_head = Some(state.candidates[0].branch.clone());
5087            state.pr_merged_head = Some(f.base.clone());
5088            state.recovery = Some(
5089                serde_json::from_value(serde_json::json!({
5090                    "from_run": "20260101-000000-abcd", "generation": 1,
5091                    "mode": "full-review"
5092                }))
5093                .unwrap(),
5094            );
5095            state.pr_recovery = Some(
5096                serde_json::from_value(serde_json::json!({
5097                    "url": "https://example.invalid/pull/1", "branch": "public",
5098                    "head": f.base, "source_run": state.id,
5099                    "instruction": "keep implementation", "approved_at": Fx::now()
5100                }))
5101                .unwrap(),
5102            );
5103            state.save_under(&f.home).unwrap();
5104            let review = reviewer(&f, &state, &f.base).await;
5105            let mut other = inactive_source(&f, "ef02", true).await;
5106            other.pr_recovery = state.pr_recovery.clone();
5107            other.save_under(&f.home).unwrap();
5108            let mut task = f.task(&[state.clone(), other]);
5109            task.pr_recovery = state.pr_recovery.clone();
5110            crate::queue::Queue::at(f.home.join("queue"))
5111                .put(&mut task)
5112                .unwrap();
5113            let run_path = f.home.join("runs").join(&state.id).join("run.json");
5114            let before = std::fs::read(&run_path).unwrap();
5115            let task_before = serde_json::to_value(&task).unwrap();
5116            let mut report = Report::default();
5117            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5118            assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5119            assert!(!review.exists());
5120            assert!(state.candidates[0].worktree.exists());
5121            assert!(
5122                git::branch_exists(&f.repo, &state.candidates[0].branch)
5123                    .await
5124                    .unwrap()
5125            );
5126            assert_eq!(std::fs::read(&run_path).unwrap(), before);
5127            assert_eq!(
5128                serde_json::to_value(
5129                    crate::queue::Queue::at(f.home.join("queue"))
5130                        .get(&task.id)
5131                        .unwrap()
5132                )
5133                .unwrap(),
5134                task_before
5135            );
5136        }
5137    }
5138
5139    #[tokio::test]
5140    async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5141        for kind in ["own-path", "active-seat", "paused-session"] {
5142            let f = Fx::new().await;
5143            let mut state = inactive_source(&f, "ef03", true).await;
5144            let review = reviewer(&f, &state, &f.base).await;
5145            let expected = match kind {
5146                "own-path" => {
5147                    state.attachments.push(review.join("source"));
5148                    "run.attachments[0]"
5149                }
5150                "active-seat" => {
5151                    state.active.insert(
5152                        "review-1".into(),
5153                        serde_json::from_value(serde_json::json!({
5154                            "node":"review", "started_at":Fx::now(), "timeout_secs":60
5155                        }))
5156                        .unwrap(),
5157                    );
5158                    "continuation lease"
5159                }
5160                _ => {
5161                    state.status = RunStatus::Blocked;
5162                    state.parked = true;
5163                    let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5164                    seat.turns = 2;
5165                    seat.captured_session = Some("saved-session".into());
5166                    state.seats.insert("review-1".into(), seat);
5167                    "resume contract"
5168                }
5169            };
5170            state.save_under(&f.home).unwrap();
5171            let mut report = Report::default();
5172            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5173            assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5174            assert!(review.exists());
5175            assert!(
5176                report.retained.iter().any(|r| r.contains(expected)),
5177                "{kind}: {report:?}"
5178            );
5179            // The execution restoration boundary leaves protected existing
5180            // seats and their sessions untouched; prep need not recreate them.
5181            restore_execution_seats(&f.home, &state).await.unwrap();
5182            assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5183        }
5184    }
5185
5186    #[tokio::test]
5187    async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5188        for kind in ["waiting", "lease", "historical", "unknown"] {
5189            let f = Fx::new().await;
5190            let state = inactive_source(&f, "ef06", true).await;
5191            let review = reviewer(&f, &state, &f.base).await;
5192            let store = crate::ask::Questions::at(f.home.join("questions"));
5193            let mut question = crate::ask::Question::new(
5194                "20260101-000000-ffff".into(),
5195                "review".into(),
5196                "review-1".into(),
5197                "fixture".into(),
5198                "fixture".into(),
5199                Vec::new(),
5200            );
5201            question.cwd = Some(review.to_string_lossy().into_owned());
5202            if kind != "waiting" {
5203                question.status = crate::ask::QuestionStatus::Abandoned;
5204            }
5205            store.put(&mut question).unwrap();
5206            if kind == "lease" {
5207                store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5208            }
5209            if kind == "unknown" {
5210                let path = store.path_of(&question.id);
5211                let mut raw: serde_json::Value =
5212                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5213                raw["future_cwd_authority"] = serde_json::Value::Null;
5214                std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5215            }
5216            let mut report = Report::default();
5217            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5218            assert_eq!(
5219                report.evicted_copies,
5220                usize::from(kind == "historical"),
5221                "{kind}: {report:?}"
5222            );
5223            assert_eq!(review.exists(), kind != "historical");
5224            if kind == "waiting" || kind == "lease" {
5225                assert!(
5226                    report
5227                        .retained
5228                        .iter()
5229                        .any(|r| r.contains("question") && r.contains("cwd")),
5230                    "{report:?}"
5231                );
5232            }
5233        }
5234    }
5235
5236    #[tokio::test]
5237    async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5238        let f = Fx::new().await;
5239        let mut state = inactive_source(&f, "ef07", true).await;
5240        let review = reviewer(&f, &state, &f.base).await;
5241        let mut report = Report::default();
5242        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5243        assert_eq!(report.evicted_copies, 1, "{report:?}");
5244        state.config.worktree.setup = vec![crate::config::SetupStep {
5245            run: Some("printf preserved > source; exit 1".into()),
5246            ..Default::default()
5247        }];
5248        state.save_under(&f.home).unwrap();
5249        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5250        let marker = f
5251            .home
5252            .join("runs")
5253            .join(&state.id)
5254            .join("artifacts/restore-review-1.pending");
5255        assert!(marker.exists());
5256        assert_eq!(
5257            std::fs::read_to_string(review.join("source")).unwrap(),
5258            "preserved"
5259        );
5260        assert!(
5261            restore_execution_seats(&f.home, &state)
5262                .await
5263                .unwrap_err()
5264                .to_string()
5265                .contains("interrupted execution restoration")
5266        );
5267        let mut retry = Report::default();
5268        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5269        assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5270        assert!(
5271            retry
5272                .retained
5273                .iter()
5274                .any(|r| r.contains("restoration checkpoint")),
5275            "{retry:?}"
5276        );
5277        assert!(marker.exists());
5278        assert_eq!(
5279            std::fs::read_to_string(review.join("source")).unwrap(),
5280            "preserved"
5281        );
5282    }
5283
5284    #[tokio::test]
5285    async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5286        for kind in [
5287            "alternates",
5288            "promisor-config",
5289            "promisor-pack",
5290            "objects-symlink",
5291        ] {
5292            let f = Fx::new().await;
5293            let state = inactive_source(&f, "ef11", true).await;
5294            let review = reviewer(&f, &state, &f.base).await;
5295            let common = crate::repo_lock::common_dir(&f.repo)
5296                .await
5297                .unwrap()
5298                .unwrap();
5299            match kind {
5300                "alternates" => {
5301                    let borrowed = f._temp.path().join("borrowed-objects");
5302                    std::fs::create_dir(&borrowed).unwrap();
5303                    std::fs::write(
5304                        common.join("objects/info/alternates"),
5305                        format!("{}\n", borrowed.display()),
5306                    )
5307                    .unwrap();
5308                }
5309                "promisor-config" => {
5310                    git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5311                        .await
5312                        .unwrap();
5313                }
5314                "promisor-pack" => {
5315                    std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5316                }
5317                _ => {
5318                    let borrowed = f._temp.path().join("borrowed-objects");
5319                    std::fs::rename(common.join("objects"), &borrowed).unwrap();
5320                    std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5321                }
5322            }
5323            let mut report = Report::default();
5324            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5325            assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5326            assert!(review.exists());
5327            assert!(
5328                report.retained.iter().any(|r| r.contains("source anchor")),
5329                "{kind}: {report:?}"
5330            );
5331        }
5332    }
5333
5334    #[tokio::test]
5335    async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5336        let mut f = Fx::new().await;
5337        crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5338        f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5339        assert_eq!(
5340            git::git(&f.repo, &["rev-list", "--count", &f.base])
5341                .await
5342                .unwrap(),
5343            "5001"
5344        );
5345        let state = inactive_source(&f, "ef10", true).await;
5346        let review = reviewer(&f, &state, &f.base).await;
5347        std::fs::write(review.join("source"), "bounded review delta").unwrap();
5348        git::git(&review, &["add", "."]).await.unwrap();
5349        let tree = git::git(&review, &["write-tree"]).await.unwrap();
5350        let head = git::git(
5351            &review,
5352            &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5353        )
5354        .await
5355        .unwrap();
5356        git::git(&review, &["checkout", "--detach", &head])
5357            .await
5358            .unwrap();
5359        let run_file = f.home.join("runs").join(&state.id).join("run.json");
5360        let run_before = std::fs::read(&run_file).unwrap();
5361        let mut report = Report::default();
5362        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5363        assert_eq!(report.evicted_copies, 1, "{report:?}");
5364        assert!(!review.exists());
5365        assert!(state.candidates[0].worktree.exists());
5366        assert!(
5367            git::branch_exists(&f.repo, &state.candidates[0].branch)
5368                .await
5369                .unwrap()
5370        );
5371        assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5372        let archive = f
5373            .home
5374            .join("runs")
5375            .join(&state.id)
5376            .join("artifacts/source-evidence")
5377            .join(format!("{head}-{}", f.base));
5378        let manifest: serde_json::Value =
5379            serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5380        assert_eq!(manifest["anchor"], f.base);
5381        assert!(
5382            std::fs::metadata(archive.join("objects.pack"))
5383                .unwrap()
5384                .len()
5385                < 16 * 1024
5386        );
5387        let pin = format!("refs/magi/source-anchors/{}", f.base);
5388        assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5389        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5390            .await
5391            .unwrap();
5392        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5393        assert!(
5394            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5395                .await
5396                .unwrap()
5397                .ok()
5398        );
5399        let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5400            .await
5401            .unwrap();
5402        // Missing, moved, symbolic and symlink pins cannot authorize import.
5403        for kind in ["missing", "moved", "symbolic", "symlink"] {
5404            git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5405                .await
5406                .unwrap();
5407            match kind {
5408                "moved" => {
5409                    git::git(&f.repo, &["update-ref", &pin, &other])
5410                        .await
5411                        .unwrap();
5412                }
5413                "symbolic" => {
5414                    git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5415                        .await
5416                        .unwrap();
5417                }
5418                "symlink" => {
5419                    let common = crate::repo_lock::common_dir(&f.repo)
5420                        .await
5421                        .unwrap()
5422                        .unwrap();
5423                    let target = f._temp.path().join("operator-anchor");
5424                    std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5425                    let file = common.join(&pin);
5426                    std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5427                    std::os::unix::fs::symlink(target, file).unwrap();
5428                }
5429                _ => {}
5430            }
5431            assert!(
5432                restore_execution_seats(&f.home, &state).await.is_err(),
5433                "{kind}"
5434            );
5435            assert!(!review.exists());
5436            assert!(
5437                !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5438                    .await
5439                    .unwrap()
5440                    .ok()
5441            );
5442            if kind == "symlink" {
5443                let common = crate::repo_lock::common_dir(&f.repo)
5444                    .await
5445                    .unwrap()
5446                    .unwrap();
5447                std::fs::remove_file(common.join(&pin)).unwrap();
5448            } else {
5449                git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5450                    .await
5451                    .unwrap();
5452            }
5453            git::git(&f.repo, &["update-ref", &pin, &f.base])
5454                .await
5455                .unwrap();
5456        }
5457        // Fixture-only ref deletion proves the pin, rather than an incidental
5458        // main/candidate branch, keeps all shared ancestors through GC.
5459        assert!(
5460            git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5461                .await
5462                .unwrap()
5463        );
5464        git::git(
5465            &f.repo,
5466            &[
5467                "update-ref",
5468                "-d",
5469                &format!("refs/heads/{}", state.candidates[0].branch),
5470            ],
5471        )
5472        .await
5473        .unwrap();
5474        git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5475            .await
5476            .unwrap();
5477        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5478            .await
5479            .unwrap();
5480        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5481        assert_eq!(
5482            git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5483                .await
5484                .unwrap(),
5485            pin
5486        );
5487        assert_eq!(
5488            git::git(&f.repo, &["rev-list", "--count", &f.base])
5489                .await
5490                .unwrap(),
5491            "5001"
5492        );
5493        restore_execution_seats(&f.home, &state).await.unwrap();
5494        assert_eq!(
5495            git::git(&review, &["rev-list", "--count", "HEAD"])
5496                .await
5497                .unwrap(),
5498            "5002"
5499        );
5500        assert_eq!(
5501            git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5502                .await
5503                .unwrap(),
5504            "false"
5505        );
5506        git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5507        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5508        let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5509        assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5510        std::fs::remove_dir_all(&f.repo).unwrap();
5511        assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5512        let inspection = f._temp.path().join("shared-history-inspection");
5513        restore_seat(&f.home, &state.id, &head, &inspection)
5514            .await
5515            .unwrap();
5516        assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5517        assert_eq!(
5518            git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5519                .await
5520                .unwrap(),
5521            "true"
5522        );
5523        assert_eq!(
5524            git::git(&inspection, &["log", "--format=%H"])
5525                .await
5526                .unwrap(),
5527            format!("{head}\n{}", f.base)
5528        );
5529        assert!(!inspection.join(".git/objects/info/alternates").exists());
5530        git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5531        git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5532    }
5533
5534    #[tokio::test]
5535    async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5536        let f = Fx::new().await;
5537        let state = inactive_source(&f, "ef08", true).await;
5538        let review = reviewer(&f, &state, &f.base).await;
5539        let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5540        let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5541            .await
5542            .unwrap();
5543        let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
5544            .await
5545            .unwrap();
5546        git::git(&review, &["checkout", "--detach", &head])
5547            .await
5548            .unwrap();
5549        let archive =
5550            crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
5551        let mut report = Report::default();
5552        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5553        assert_eq!(report.evicted_copies, 0, "{report:?}");
5554        assert!(review.exists());
5555        assert!(
5556            report
5557                .retained
5558                .iter()
5559                .any(|r| r.contains("incomplete archived commit ancestry")),
5560            "{report:?}"
5561        );
5562        // Simulate a historical checkout already removed by the old writer.
5563        assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
5564        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5565            .await
5566            .unwrap();
5567        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5568        for oid in [&parent, &head] {
5569            assert!(
5570                !git::git_raw(&f.repo, &["cat-file", "-e", oid])
5571                    .await
5572                    .unwrap()
5573                    .ok()
5574            );
5575        }
5576        let patch = source_diff(&archive).await.unwrap();
5577        assert!(
5578            crate::fold_evidence::import_snapshot(&archive, &f.repo)
5579                .await
5580                .is_err()
5581        );
5582        assert!(
5583            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5584                .await
5585                .unwrap()
5586                .ok()
5587        );
5588        git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5589        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5590        let inspection = f._temp.path().join("legacy-tree-inspection");
5591        restore_source(&archive, &inspection).await.unwrap();
5592        assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
5593        assert!(
5594            !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
5595                .await
5596                .unwrap()
5597                .ok()
5598        );
5599        git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5600        git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5601        assert_eq!(source_diff(&archive).await.unwrap(), patch);
5602        assert!(state.candidates[0].worktree.exists());
5603    }
5604
5605    #[tokio::test]
5606    async fn source_eviction_retains_ancestry_above_expanded_budget() {
5607        let f = Fx::new().await;
5608        let state = inactive_source(&f, "ef09", true).await;
5609        let review = reviewer(&f, &state, &f.base).await;
5610        std::fs::File::create(review.join("ancestor-only"))
5611            .unwrap()
5612            .set_len(64 * 1024 * 1024 + 1)
5613            .unwrap();
5614        git::git(&review, &["add", "."]).await.unwrap();
5615        let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5616        let parent = git::git(
5617            &review,
5618            &["commit-tree", &parent_tree, "-m", "large ancestor"],
5619        )
5620        .await
5621        .unwrap();
5622        std::fs::remove_file(review.join("ancestor-only")).unwrap();
5623        git::git(&review, &["add", "."]).await.unwrap();
5624        let tree = git::git(&review, &["write-tree"]).await.unwrap();
5625        let head = git::git(
5626            &review,
5627            &[
5628                "commit-tree",
5629                &tree,
5630                "-p",
5631                &parent,
5632                "-m",
5633                "small final tree",
5634            ],
5635        )
5636        .await
5637        .unwrap();
5638        git::git(&review, &["checkout", "--detach", &head])
5639            .await
5640            .unwrap();
5641        let mut report = Report::default();
5642        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5643        assert_eq!(report.evicted_copies, 0, "{report:?}");
5644        assert!(
5645            report
5646                .retained
5647                .iter()
5648                .any(|r| r.contains("expanded object limit exceeded")),
5649            "{report:?}"
5650        );
5651        assert!(review.exists());
5652        assert!(state.candidates[0].worktree.exists());
5653    }
5654
5655    #[tokio::test]
5656    async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
5657        let f = Fx::new().await;
5658        let state = inactive_source(&f, "ef04", true).await;
5659        let review = reviewer(&f, &state, &f.base).await;
5660        std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
5661        git::git(&review, &["add", "."]).await.unwrap();
5662        let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5663        let parent = git::git(
5664            &review,
5665            &["commit-tree", &parent_tree, "-m", "unrelated parent"],
5666        )
5667        .await
5668        .unwrap();
5669        std::fs::remove_file(review.join("ancestor-only")).unwrap();
5670        std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
5671        git::git(&review, &["add", "."]).await.unwrap();
5672        let tree = git::git(&review, &["write-tree"]).await.unwrap();
5673        let head = git::git(
5674            &review,
5675            &[
5676                "commit-tree",
5677                &tree,
5678                "-p",
5679                &parent,
5680                "-m",
5681                "unrelated review checkpoint",
5682            ],
5683        )
5684        .await
5685        .unwrap();
5686        git::git(&review, &["checkout", "--detach", &head])
5687            .await
5688            .unwrap();
5689        let mut report = Report::default();
5690        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5691        assert_eq!(report.evicted_copies, 1, "{report:?}");
5692        let ledger = f
5693            .home
5694            .join("runs")
5695            .join(&state.id)
5696            .join("artifacts/eviction-review-1.json");
5697        let original = std::fs::read(&ledger).unwrap();
5698        let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
5699        assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
5700        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5701            .await
5702            .unwrap();
5703        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5704        assert!(
5705            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5706                .await
5707                .unwrap()
5708                .ok()
5709        );
5710        assert!(
5711            !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
5712                .await
5713                .unwrap()
5714                .ok()
5715        );
5716        assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5717        assert!(
5718            seat_heads(&f.home, &state.id)
5719                .await
5720                .unwrap()
5721                .contains_key(&head)
5722        );
5723        restore_execution_seats(&f.home, &state).await.unwrap();
5724        assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
5725        assert_eq!(
5726            git::git(&review, &["log", "--format=%H"]).await.unwrap(),
5727            format!("{head}\n{parent}")
5728        );
5729        git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5730        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5731        let inspection = f._temp.path().join("complete-history-inspection");
5732        restore_seat(&f.home, &state.id, &head, &inspection)
5733            .await
5734            .unwrap();
5735        assert_eq!(
5736            git::git(&inspection, &["log", "--format=%H"])
5737                .await
5738                .unwrap(),
5739            format!("{head}\n{parent}")
5740        );
5741        git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5742        git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5743
5744        assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
5745        assert!(
5746            !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
5747                .await
5748                .unwrap()
5749                .ok()
5750        );
5751        assert_eq!(
5752            std::fs::read_to_string(review.join("source")).unwrap(),
5753            "unrelated review checkpoint"
5754        );
5755        std::fs::write(review.join("operator-notes"), "keep").unwrap();
5756        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5757        assert_eq!(
5758            std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5759            "keep"
5760        );
5761        std::fs::remove_file(review.join("operator-notes")).unwrap();
5762        restore_execution_seats(&f.home, &state).await.unwrap();
5763        let mut second = Report::default();
5764        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
5765        assert_eq!(second.evicted_copies, 1, "{second:?}");
5766        assert_eq!(std::fs::read(&ledger).unwrap(), original);
5767        assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
5768        // A historical v1 fixture has no explicit base field. Neither schema
5769        // may reinterpret its immutable archive when the owning run advances.
5770        let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
5771        legacy["version"] = 1.into();
5772        legacy.as_object_mut().unwrap().remove("base");
5773        std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
5774        let mut advanced = state.clone();
5775        advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5776            .await
5777            .unwrap();
5778        advanced.save_under(&f.home).unwrap();
5779        assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5780        // Corruption blocks execution before a replacement checkout is made.
5781        let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
5782        let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
5783        let patch = std::fs::read(archive.join("change.patch")).unwrap();
5784        std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
5785        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5786        assert!(!review.exists());
5787        std::fs::write(archive.join("change.patch"), patch).unwrap();
5788        std::fs::create_dir(&review).unwrap();
5789        std::fs::write(review.join("operator-notes"), "keep").unwrap();
5790        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5791        assert_eq!(
5792            std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5793            "keep"
5794        );
5795    }
5796
5797    #[tokio::test]
5798    async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
5799        use std::os::unix::fs::PermissionsExt;
5800        let mut f = Fx::new().await;
5801        std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
5802        git::git(&f.repo, &["add", "."]).await.unwrap();
5803        git::git(&f.repo, &["commit", "-m", "binary base"])
5804            .await
5805            .unwrap();
5806        f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5807        let mut state = inactive_source(&f, "ec01", true).await;
5808        let candidate = state.candidates[0].worktree.clone();
5809        let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5810        historical_review(&mut state, &earlier);
5811        let review = reviewer(&f, &state, &earlier).await;
5812        std::fs::remove_file(candidate.join("source")).unwrap();
5813        std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5814        std::fs::write(candidate.join("script"), "run\n").unwrap();
5815        std::fs::set_permissions(
5816            candidate.join("script"),
5817            std::fs::Permissions::from_mode(0o755),
5818        )
5819        .unwrap();
5820        git::git(&candidate, &["add", "."]).await.unwrap();
5821        git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5822            .await
5823            .unwrap();
5824        let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5825        let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5826        let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5827            .await
5828            .unwrap();
5829        historical_review(&mut state, &head);
5830        historical_review(&mut state, &head);
5831        state.status = RunStatus::Superseded;
5832        state.save_under(&f.home).unwrap();
5833        let task = f.task(std::slice::from_ref(&state));
5834        let before = serde_json::to_value(&state).unwrap();
5835        git::git(&review, &["reset", "--hard", &head])
5836            .await
5837            .unwrap();
5838        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5839        assert_eq!(report.evicted_copies, 1, "{report:?}");
5840        assert!(!review.exists());
5841        assert!(candidate.exists(), "selected implementation survives");
5842        assert!(report.source_unlinked_logical_bytes > 0);
5843        assert!(report.source_unlinked_allocated_bytes.is_some());
5844        assert!(report.source_evidence_added_allocated_bytes.is_some());
5845        assert_eq!(
5846            serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5847            before
5848        );
5849        assert_eq!(
5850            crate::queue::Queue::at(f.home.join("queue"))
5851                .get(&task.id)
5852                .unwrap()
5853                .runs,
5854            task.runs
5855        );
5856        let archive = f
5857            .home
5858            .join("runs")
5859            .join(&state.id)
5860            .join("artifacts/source-evidence")
5861            .join(format!("{head}-{}", f.base));
5862        let patch = source_diff(&archive).await.unwrap();
5863        assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5864        assert!(String::from_utf8_lossy(&patch).contains("100755"));
5865        assert!(
5866            git::worktree_remove_clean(&f.repo, &candidate)
5867                .await
5868                .unwrap()
5869        );
5870        git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5871            .await
5872            .unwrap();
5873        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5874            .await
5875            .unwrap();
5876        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5877        assert!(
5878            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5879                .await
5880                .unwrap()
5881                .ok()
5882        );
5883        // Even the remaining base repository is unavailable to the reader.
5884        std::fs::remove_dir_all(&f.repo).unwrap();
5885        assert_eq!(source_diff(&archive).await.unwrap(), patch);
5886        let restored = f._temp.path().join("restored");
5887        restore_source(&archive, &restored).await.unwrap();
5888        assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5889        assert_eq!(
5890            std::fs::read(restored.join("binary")).unwrap(),
5891            b"new\0binary\xfe"
5892        );
5893        assert!(!restored.join("source").exists());
5894        assert_ne!(
5895            std::fs::metadata(restored.join("script"))
5896                .unwrap()
5897                .permissions()
5898                .mode()
5899                & 0o111,
5900            0
5901        );
5902        assert!(restore_source(&archive, &restored).await.is_err());
5903        // Reconstruct from the archived base plus the displayed binary patch,
5904        // rather than relying only on checking out the archived final tree.
5905        git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5906            .await
5907            .unwrap();
5908        let patch_file = f._temp.path().join("saved.patch");
5909        std::fs::write(&patch_file, &patch).unwrap();
5910        git::git(
5911            &restored,
5912            &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5913        )
5914        .await
5915        .unwrap();
5916        assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5917        assert_eq!(
5918            std::fs::read(restored.join("binary")).unwrap(),
5919            b"new\0binary\xfe"
5920        );
5921    }
5922
5923    #[tokio::test]
5924    async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5925        let f = Fx::new().await;
5926        let state = inactive_source(&f, "ec10", true).await;
5927        let review = reviewer(&f, &state, &f.base).await;
5928        let mut successor = inactive_source(&f, "ec11", true).await;
5929        successor.retired_to = Some(state.id.clone());
5930        successor.save_under(&f.home).unwrap();
5931        f.task(&[state.clone(), successor]);
5932
5933        let mut tips = Vec::new();
5934        for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5935            std::fs::write(review.join("source"), contents).unwrap();
5936            git::git(&review, &["add", "."]).await.unwrap();
5937            let tree = git::git(&review, &["write-tree"]).await.unwrap();
5938            let tip = git::git(
5939                &review,
5940                &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5941            )
5942            .await
5943            .unwrap();
5944            git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5945            tips.push(tip);
5946            git::git(&review, &["reset", "--hard", &f.base])
5947                .await
5948                .unwrap();
5949        }
5950        // Keep one tip exclusively in ORIG_HEAD, the other in the HEAD log.
5951        git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5952            .await
5953            .unwrap();
5954        git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5955            .await
5956            .unwrap();
5957        let mut report = Report::default();
5958        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5959        assert_eq!(report.evicted_copies, 1, "{report:?}");
5960        assert!(!review.exists());
5961        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5962            .await
5963            .unwrap();
5964        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5965        for tip in &tips {
5966            assert!(
5967                !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5968                    .await
5969                    .unwrap()
5970                    .ok()
5971            );
5972        }
5973        // The run-oriented interface also works without the source object store.
5974        std::fs::remove_dir_all(&f.repo).unwrap();
5975        let shown = seat_evidence(&f.home, &state.id).await.unwrap();
5976        for (i, tip) in tips.iter().enumerate() {
5977            assert!(
5978                shown
5979                    .iter()
5980                    .any(|row| row.contains(tip) && row.contains("patch_bytes="))
5981            );
5982            let destination = f._temp.path().join(format!("restore-history-{i}"));
5983            restore_seat(&f.home, &state.id, tip, &destination)
5984                .await
5985                .unwrap();
5986            assert_eq!(
5987                std::fs::read_to_string(destination.join("source")).unwrap(),
5988                ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
5989            );
5990        }
5991        assert!(
5992            restore_seat(
5993                &f.home,
5994                &state.id,
5995                "unknown",
5996                &f._temp.path().join("missing")
5997            )
5998            .await
5999            .is_err()
6000        );
6001    }
6002
6003    #[tokio::test]
6004    async fn source_eviction_revalidates_archive_immediately_before_unlink() {
6005        fn corrupt(path: &Path) -> Result<()> {
6006            let root = path.parent().unwrap();
6007            // Locate the fixture's durable artifact store through its run record
6008            // path supplied below by a test-only hook in the artifact tree.
6009            let marker = root.join("archive-to-corrupt");
6010            let archive = std::fs::read_to_string(&marker)?;
6011            std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
6012            Ok(())
6013        }
6014        let f = Fx::new().await;
6015        let state = inactive_source(&f, "ec12", true).await;
6016        let review = reviewer(&f, &state, &f.base).await;
6017        let archive = f
6018            .home
6019            .join("runs")
6020            .join(&state.id)
6021            .join("artifacts/source-evidence")
6022            .join(format!("{}-{}", f.base, f.base));
6023        std::fs::write(
6024            state.worktree_root().join("archive-to-corrupt"),
6025            archive.to_str().unwrap(),
6026        )
6027        .unwrap();
6028        let mut report = Report::default();
6029        copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
6030        assert_eq!(report.evicted_copies, 0, "{report:?}");
6031        assert!(review.exists());
6032        assert!(
6033            report
6034                .retained
6035                .iter()
6036                .any(|r| r.contains("corrupt snapshot")),
6037            "{report:?}"
6038        );
6039    }
6040
6041    #[tokio::test]
6042    async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6043        for kind in [
6044            "dirty",
6045            "hidden",
6046            "ignored",
6047            "empty-directory",
6048            "untracked",
6049            "borrowed",
6050            "borrowed-symlink",
6051            "lease",
6052            "owner",
6053            "driver",
6054            "task-pin",
6055            "task-replacement",
6056            "malformed-orig-head",
6057            "grace",
6058            "resumable",
6059            "archive-symlink",
6060            "root-symlink",
6061            "checkpoint",
6062            "symlink",
6063            "retained",
6064            "unreadable",
6065            "unknown-field",
6066            "unknown-task-field",
6067        ] {
6068            let f = Fx::new().await;
6069            let mut state = inactive_source(&f, "ec02", true).await;
6070            let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6071                .await
6072                .unwrap();
6073            historical_review(&mut state, &head);
6074            let review = reviewer(&f, &state, &head).await;
6075            let mut output = None;
6076            let mut owner = None;
6077            let mut driver = None;
6078            let mut disk = Fx::disk();
6079            match kind {
6080                "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6081                "hidden" => {
6082                    git::git(&review, &["update-index", "--assume-unchanged", "source"])
6083                        .await
6084                        .unwrap();
6085                    std::fs::write(review.join("source"), "notes").unwrap();
6086                }
6087                "ignored" => {
6088                    std::fs::create_dir(review.join("scratch")).unwrap();
6089                    std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6090                }
6091                "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6092                "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6093                "borrowed" => {
6094                    let mut other = f.run("ec03", true).await;
6095                    other.candidates[0].worktree = review.clone();
6096                    other.save_under(&f.home).unwrap();
6097                }
6098                "borrowed-symlink" => {
6099                    let alias = f._temp.path().join("borrowed-alias");
6100                    std::os::unix::fs::symlink(&review, &alias).unwrap();
6101                    let mut other = f.run("ec03", true).await;
6102                    other.candidates[0].worktree = alias;
6103                    other.save_under(&f.home).unwrap();
6104                }
6105                "lease" => output = OutputLease::borrow(&review).unwrap(),
6106                "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6107                "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6108                "task-pin" => {
6109                    let mut task = f.task(std::slice::from_ref(&state));
6110                    task.resume_override = Some(crate::queue::OperatorResume {
6111                        question_id: "fixture-pin".into(),
6112                        at: Fx::now(),
6113                        conductor_rehold: None,
6114                        forced: true,
6115                        pinned_run: Some(state.id.clone()),
6116                    });
6117                    crate::queue::Queue::at(f.home.join("queue"))
6118                        .put(&mut task)
6119                        .unwrap();
6120                }
6121                "task-replacement" => {
6122                    let mut task = f.task(std::slice::from_ref(&state));
6123                    task.replacement = Some(crate::merge_decision::Replacement {
6124                        question: "fixture-replacement".into(),
6125                        run: state.id.clone(),
6126                        pr: "https://example.invalid/pull/1".into(),
6127                        agent: "worker".into(),
6128                        context: String::new(),
6129                    });
6130                    crate::queue::Queue::at(f.home.join("queue"))
6131                        .put(&mut task)
6132                        .unwrap();
6133                }
6134                "malformed-orig-head" => {
6135                    let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6136                        .await
6137                        .unwrap();
6138                    std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6139                        .unwrap();
6140                }
6141                "grace" => disk.fold_grace_secs = 3600,
6142                "resumable" => state.status = RunStatus::Blocked,
6143                "archive-symlink" => {
6144                    let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6145                    std::fs::create_dir_all(&artifacts).unwrap();
6146                    let outside = f._temp.path().join("outside");
6147                    std::fs::create_dir(&outside).unwrap();
6148                    std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6149                        .unwrap();
6150                }
6151                "root-symlink" => {
6152                    let root = state.worktree_root();
6153                    let outside = f._temp.path().join("outside-root");
6154                    std::fs::rename(&root, &outside).unwrap();
6155                    std::os::unix::fs::symlink(&outside, &root).unwrap();
6156                }
6157                "checkpoint" => {
6158                    let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6159                        .await
6160                        .unwrap();
6161                    std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6162                }
6163                "symlink" => {
6164                    std::fs::remove_file(review.join("source")).unwrap();
6165                    std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6166                }
6167                "retained" => std::fs::write(
6168                    f.home
6169                        .join("runs")
6170                        .join(&state.id)
6171                        .join("retain-recovery-sources"),
6172                    "owner",
6173                )
6174                .unwrap(),
6175                "unreadable" => {
6176                    std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6177                }
6178                "unknown-field" | "unknown-task-field" => {}
6179                _ => unreachable!(),
6180            }
6181            state.save_under(&f.home).unwrap();
6182            if kind == "unknown-field" {
6183                let path = f.home.join("runs").join(&state.id).join("run.json");
6184                let mut raw: serde_json::Value =
6185                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6186                raw["future_source_owner"] = serde_json::json!({"path": review});
6187                std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6188            }
6189            if kind == "unknown-task-field" {
6190                let task = f.task(std::slice::from_ref(&state));
6191                let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6192                let mut raw: serde_json::Value =
6193                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6194                raw["future_source_owner"] = serde_json::json!({"path": review});
6195                std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6196            }
6197            let mut report = Report::default();
6198            copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6199            assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6200            assert!(review.exists(), "{kind}: {report:?}");
6201            assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6202            drop(output);
6203            drop(owner);
6204            drop(driver);
6205        }
6206    }
6207
6208    #[tokio::test]
6209    async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6210        let f = Fx::new().await;
6211        let state = inactive_source(&f, "ec04", true).await;
6212        let review = reviewer(&f, &state, &f.base).await;
6213        let mut report = Report::default();
6214        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6215        assert_eq!(report.evicted_copies, 1, "{report:?}");
6216        let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6217        let ledger = artifacts.join("eviction-review-1.json");
6218        let mut record: serde_json::Value =
6219            serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6220        record["phase"] = "pending".into();
6221        std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6222        let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6223        let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6224        std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6225        let mut blocked = Report::default();
6226        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6227        assert!(
6228            blocked
6229                .retained
6230                .iter()
6231                .any(|r| r.contains("corrupt snapshot")),
6232            "{blocked:?}"
6233        );
6234        assert_eq!(
6235            serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6236                ["phase"],
6237            "pending"
6238        );
6239        assert!(
6240            restore_source(&archive, &f._temp.path().join("bad-restore"))
6241                .await
6242                .is_err()
6243        );
6244        std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6245        let mut retried = Report::default();
6246        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6247        assert_eq!(
6248            serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6249                ["phase"],
6250            "evicted"
6251        );
6252        assert_eq!(retried.evicted_copies, 0);
6253        assert!(!review.exists());
6254        // A stopped archive publisher can resume its immutable staging files.
6255        let pending = archive.with_extension("pending");
6256        std::fs::rename(&archive, &pending).unwrap();
6257        assert_eq!(
6258            crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6259                .await
6260                .unwrap(),
6261            archive
6262        );
6263        assert!(!pending.exists());
6264    }
6265
6266    #[tokio::test]
6267    async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6268        let f = Fx::new().await;
6269        let mut empty = inactive_source(&f, "ec05", false).await;
6270        historical_review(&mut empty, &f.base);
6271        empty.status = RunStatus::Superseded;
6272        empty.save_under(&f.home).unwrap();
6273        let source = inactive_source(&f, "ec06", true).await;
6274        let task = f.task(&[empty.clone(), source.clone()]);
6275        let before = serde_json::to_value(&empty).unwrap();
6276        let mut report = Report::default();
6277        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6278        assert_eq!(report.evicted_copies, 1, "{report:?}");
6279        assert!(!empty.candidates[0].worktree.exists());
6280        assert!(source.candidates[0].worktree.exists());
6281        assert!(
6282            git::branch_exists(&f.repo, &empty.candidates[0].branch)
6283                .await
6284                .unwrap()
6285        );
6286        assert_eq!(
6287            serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6288            before
6289        );
6290        let mut retry = Report::default();
6291        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6292        assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6293        assert!(!empty.worktree_root().exists());
6294        assert_eq!(
6295            crate::queue::Queue::at(f.home.join("queue"))
6296                .get(&task.id)
6297                .unwrap()
6298                .runs,
6299            task.runs
6300        );
6301    }
6302    #[tokio::test]
6303    async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6304        fn interrupt(_: &Path) -> Result<()> {
6305            anyhow::bail!("fixture interruption before unlink")
6306        }
6307        fn vanish(path: &Path) -> Result<()> {
6308            std::fs::remove_dir_all(path)?;
6309            anyhow::bail!("fixture interruption after checkout unlink")
6310        }
6311        for stage in [
6312            "intent",
6313            "missing-registration",
6314            "changed-admin",
6315            "missing-archive",
6316            "wrong-identity",
6317        ] {
6318            let f = Fx::new().await;
6319            let state = inactive_source(&f, "ec07", true).await;
6320            let review = reviewer(&f, &state, &f.base).await;
6321            let admin = PathBuf::from(
6322                git::git(&review, &["rev-parse", "--absolute-git-dir"])
6323                    .await
6324                    .unwrap(),
6325            );
6326            let mut interrupted = Report::default();
6327            copies::evict_with(
6328                &f.home,
6329                &Fx::disk(),
6330                Fx::now(),
6331                &mut interrupted,
6332                if stage == "missing-registration" || stage == "changed-admin" {
6333                    vanish
6334                } else {
6335                    interrupt
6336                },
6337            )
6338            .await;
6339            assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6340            let ledger = f
6341                .home
6342                .join("runs")
6343                .join(&state.id)
6344                .join("artifacts/eviction-review-1.json");
6345            let record: serde_json::Value =
6346                serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6347            assert_eq!(record["phase"], "pending");
6348            if stage == "changed-admin" {
6349                std::fs::write(admin.join("owner-note"), "keep").unwrap();
6350            }
6351            if stage == "missing-archive" {
6352                std::fs::remove_file(
6353                    Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6354                )
6355                .unwrap();
6356            }
6357            if stage == "wrong-identity" {
6358                let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6359                    .await
6360                    .unwrap();
6361                let other =
6362                    crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6363                        .await
6364                        .unwrap();
6365                let original = Path::new(record["evidence"].as_str().unwrap());
6366                for name in [
6367                    "manifest.json",
6368                    "objects.pack",
6369                    "change.patch",
6370                    "change.index",
6371                ] {
6372                    std::fs::copy(other.join(name), original.join(name)).unwrap();
6373                }
6374            }
6375            let mut retried = Report::default();
6376            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6377            let phase =
6378                serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6379                    .unwrap()["phase"]
6380                    .clone();
6381            if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6382                assert_eq!(phase, "pending", "{stage}: {retried:?}");
6383                assert!(!retried.retained.is_empty());
6384                if stage == "changed-admin" {
6385                    assert!(admin.join("owner-note").exists());
6386                } else {
6387                    assert!(review.exists());
6388                }
6389            } else {
6390                assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6391                assert!(!review.exists());
6392                assert!(!admin.exists());
6393            }
6394        }
6395    }
6396    #[tokio::test]
6397    async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
6398        let f = Fx::new().await;
6399        let mut runs = Vec::new();
6400        let mut paths = Vec::new();
6401        for tag in ["ec10", "ec11", "ec12", "ec13"] {
6402            let state = inactive_source(&f, tag, true).await;
6403            for name in ["judge-1", "advisor-1"] {
6404                let path = state.worktree_root().join(name);
6405                git::worktree_add_detached(&f.repo, &path, &f.base)
6406                    .await
6407                    .unwrap();
6408                paths.push(path);
6409            }
6410            runs.push(state);
6411        }
6412        let task = f.task(&runs);
6413        let mut report = Report::default();
6414        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6415        assert_eq!(report.evicted_copies, 5, "{report:?}");
6416        let mut next = Report::default();
6417        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6418        assert_eq!(next.evicted_copies, 3, "{next:?}");
6419        assert!(paths.iter().all(|p| !p.exists()));
6420        assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
6421        assert_eq!(
6422            crate::queue::Queue::at(f.home.join("queue"))
6423                .get(&task.id)
6424                .unwrap()
6425                .runs,
6426            task.runs
6427        );
6428        let limited = Fx::new().await;
6429        let state = inactive_source(&limited, "ec15", true).await;
6430        let copy = reviewer(&limited, &state, &limited.base).await;
6431        let evidence = limited
6432            .home
6433            .join("runs")
6434            .join(&state.id)
6435            .join("artifacts/source-evidence");
6436        std::fs::create_dir_all(&evidence).unwrap();
6437        let retained = evidence.join("reserved-evidence");
6438        std::fs::File::create(&retained)
6439            .unwrap()
6440            .set_len(64 * 1024 * 1024)
6441            .unwrap();
6442        let mut blocked = Report::default();
6443        copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6444        assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
6445        assert!(copy.exists());
6446        assert_eq!(
6447            std::fs::metadata(&retained).unwrap().len(),
6448            64 * 1024 * 1024
6449        );
6450        assert!(
6451            blocked
6452                .retained
6453                .iter()
6454                .any(|r| r.contains("per-run storage limit")),
6455            "{blocked:?}"
6456        );
6457    }
6458    #[tokio::test]
6459    async fn source_eviction_migrates_historical_records_without_rewriting_them() {
6460        let f = Fx::new().await;
6461        let mut state = inactive_source(&f, "ec14", true).await;
6462        historical_review(&mut state, &f.base);
6463        historical_review(&mut state, &f.base);
6464        state.status = RunStatus::Superseded;
6465        state.save_under(&f.home).unwrap();
6466        let review = reviewer(&f, &state, &f.base).await;
6467        let task = f.task(std::slice::from_ref(&state));
6468        let path = f.home.join("runs").join(&state.id).join("run.json");
6469        let mut raw: serde_json::Value =
6470            serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6471        raw["schema"] = 10.into();
6472        raw.as_object_mut().unwrap().remove("output_activity_at");
6473        // Historical writers persisted null/empty defaults that current serde
6474        // omits. These are known fields, not unknown recovery authorities.
6475        raw["resource_wait"] = serde_json::Value::Null;
6476        raw["resource_block"] = serde_json::Value::Null;
6477        raw["driver_claimed"] = serde_json::Value::Null;
6478        raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6479        // Actual legacy spelling accepted by Graph's serde alias.
6480        let graph = raw["config"]["graph"].as_object_mut().unwrap();
6481        let count = graph.remove("implementers").unwrap();
6482        graph.insert("candidates".into(), count);
6483        let queue = crate::queue::Queue::at(f.home.join("queue"));
6484        let task_path = queue.path_of(&task.id);
6485        let mut task_raw: serde_json::Value =
6486            serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
6487        task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6488        let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
6489        std::fs::write(&task_path, &task_historical).unwrap();
6490        let historical = serde_json::to_vec_pretty(&raw).unwrap();
6491        std::fs::write(&path, &historical).unwrap();
6492        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6493        assert_eq!(report.evicted_copies, 1, "{report:?}");
6494        assert!(!review.exists());
6495        assert_eq!(std::fs::read(&path).unwrap(), historical);
6496        assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
6497        assert!(state.candidates[0].worktree.exists());
6498        assert_eq!(
6499            crate::queue::Queue::at(f.home.join("queue"))
6500                .get(&task.id)
6501                .unwrap()
6502                .runs,
6503            task.runs
6504        );
6505    }
6506    #[tokio::test]
6507    async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
6508        let f = Fx::new().await;
6509        let state = inactive_source(&f, "ed01", true).await;
6510        let review = reviewer(&f, &state, &f.base).await;
6511        std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
6512        git::git(&review, &["add", "."]).await.unwrap();
6513        let tree = git::git(&review, &["write-tree"]).await.unwrap();
6514        let tip = git::git(
6515            &review,
6516            &[
6517                "commit-tree",
6518                &tree,
6519                "-p",
6520                &f.base,
6521                "-m",
6522                "temporary detached source",
6523            ],
6524        )
6525        .await
6526        .unwrap();
6527        git::git(&review, &["checkout", "--detach", &tip])
6528            .await
6529            .unwrap();
6530        git::git(&review, &["checkout", "--detach", &f.base])
6531            .await
6532            .unwrap();
6533        git::git(&review, &["reflog", "delete", "HEAD@{1}"])
6534            .await
6535            .unwrap();
6536        git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
6537            .await
6538            .unwrap();
6539        let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
6540            .await
6541            .unwrap();
6542        assert!(!shown.lines().any(|oid| oid == tip));
6543        let admin = PathBuf::from(
6544            git::git(&review, &["rev-parse", "--absolute-git-dir"])
6545                .await
6546                .unwrap(),
6547        );
6548        let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
6549        assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
6550        fn interrupt(_: &Path) -> Result<()> {
6551            anyhow::bail!("fixture interruption before unlink")
6552        }
6553        let mut pending = Report::default();
6554        copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
6555        assert!(review.exists());
6556        let ledger = f
6557            .home
6558            .join("runs")
6559            .join(&state.id)
6560            .join("artifacts/eviction-review-1.json");
6561        let complete = std::fs::read(&ledger).unwrap();
6562        let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
6563        incomplete["history"]
6564            .as_object_mut()
6565            .unwrap()
6566            .remove(&tip)
6567            .unwrap();
6568        std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
6569        let mut blocked = Report::default();
6570        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6571        assert!(review.exists());
6572        assert!(
6573            blocked
6574                .retained
6575                .iter()
6576                .any(|reason| reason.contains("incomplete reflog checkpoint")),
6577            "{blocked:?}"
6578        );
6579        std::fs::write(&ledger, complete).unwrap();
6580        let mut report = Report::default();
6581        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6582        assert_eq!(report.evicted_copies, 1, "{report:?}");
6583        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6584            .await
6585            .unwrap();
6586        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6587        assert!(
6588            !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
6589                .await
6590                .unwrap()
6591                .ok()
6592        );
6593        std::fs::remove_dir_all(&f.repo).unwrap();
6594        let destination = f._temp.path().join("restore-old-only");
6595        restore_seat(&f.home, &state.id, &tip, &destination)
6596            .await
6597            .unwrap();
6598        assert_eq!(
6599            std::fs::read(destination.join("old-only")).unwrap(),
6600            b"old-only\0binary\xff"
6601        );
6602    }
6603
6604    #[tokio::test]
6605    async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
6606        fn interrupt(_: &Path) -> Result<()> {
6607            anyhow::bail!("fixture interruption before unlink")
6608        }
6609        fn vanish(path: &Path) -> Result<()> {
6610            std::fs::remove_dir_all(path)?;
6611            anyhow::bail!("fixture interruption before unregistration")
6612        }
6613        for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
6614            let f = Fx::new().await;
6615            let mut state = inactive_source(&f, "ed02", true).await;
6616            state.driver_exited = false;
6617            state.driver_claimed = Some(true);
6618            state.driver_pid = Some(std::process::id());
6619            state.driver_started_at = crate::proc::process_started_at(std::process::id());
6620            state.begin_pass();
6621            state.status = RunStatus::Superseded;
6622            state.save_under(&f.home).unwrap();
6623            let review = reviewer(&f, &state, &f.base).await;
6624            // The same matching identity written by DriverGuard::record, left
6625            // behind when a process exits without its cleanup guard running.
6626            let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
6627                "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
6628            std::fs::write(
6629                f.home.join("runs").join(&state.id).join("driver.json"),
6630                serde_json::to_vec(&identity).unwrap(),
6631            )
6632            .unwrap();
6633            let live = if stage == "live-driver" {
6634                Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
6635            } else {
6636                None
6637            };
6638            assert_eq!(
6639                state.liveness_under(&f.home, false),
6640                if live.is_some() {
6641                    Liveness::Live
6642                } else {
6643                    Liveness::Dead
6644                }
6645            );
6646            let mut report = Report::default();
6647            if stage == "intent" || stage == "missing-registration" {
6648                copies::evict_with(
6649                    &f.home,
6650                    &Fx::disk(),
6651                    Fx::now(),
6652                    &mut report,
6653                    if stage == "intent" { interrupt } else { vanish },
6654                )
6655                .await;
6656                assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
6657                let mut retry = Report::default();
6658                copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6659                assert!(!review.exists(), "{stage}: {retry:?}");
6660                let ledger = f
6661                    .home
6662                    .join("runs")
6663                    .join(&state.id)
6664                    .join("artifacts/eviction-review-1.json");
6665                let receipt: serde_json::Value =
6666                    serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
6667                assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
6668            } else {
6669                copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6670                assert_eq!(
6671                    report.evicted_copies,
6672                    usize::from(live.is_none()),
6673                    "{stage}: {report:?}"
6674                );
6675                assert_eq!(review.exists(), live.is_some());
6676            }
6677            drop(live);
6678            assert!(
6679                !RunState::load_under(&state.id, &f.home)
6680                    .unwrap()
6681                    .driver_exited
6682            );
6683        }
6684    }
6685}