Skip to main content

magi/
retention.rs

1//! Bounded reclamation of retained attempts and Cargo outputs. Source recovery
2//! and build eviction have separate authority: evicting a target never retires a run.
3mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7    let artifacts = home.join("runs").join(&state.id).join("artifacts");
8    if !artifacts.try_exists()? {
9        return Ok(false);
10    }
11    ensure!(
12        plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13        "noncanonical artifacts"
14    );
15    for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16        ensure!(i < 4096, "eviction record inspection limit exceeded");
17        let name = entry?.file_name();
18        let name = name.to_string_lossy();
19        if name.starts_with("eviction-") && name.ends_with(".json") {
20            return Ok(true);
21        }
22    }
23    Ok(false)
24}
25
26/// Re-materialize evicted read seats before execution can consume their paths.
27/// Caller owns the run's driver/owner leases and repository coordination.
28pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29    copies::restore_execution_seats(home, state).await
30}
31
32/// Read the binary diff after verifying its independent source archive.
33pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34    crate::fold_evidence::validate_snapshot(evidence).await?;
35    Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38/// Restore immutable checkout evidence into a new inspection repository.
39pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40    crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43/// List archived checkout tips for a run, validating their durable evidence.
44pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
45    copies::seat_evidence(home, id).await
46}
47
48/// Validated archived heads and seat labels for the run's evidence browser.
49pub async fn seat_heads(
50    home: &Path,
51    id: &str,
52) -> Result<std::collections::BTreeMap<String, String>> {
53    copies::seat_heads(home, id).await
54}
55
56/// Read a recorded seat's diff without relying on its checkout or Git objects.
57pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
58    copies::seat_diff(home, id, head).await
59}
60
61/// Restore a recorded checkout tip without changing run or recovery authority.
62pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
63    copies::restore_seat(home, id, head, destination).await
64}
65
66use crate::{
67    git,
68    proc::Quiet as _,
69    run::{Liveness, RunState},
70};
71use anyhow::{Context, Result, ensure};
72use jiff::Timestamp;
73use serde::{Deserialize, Serialize};
74use std::{
75    fs::{File, OpenOptions},
76    path::{Path, PathBuf},
77};
78
79/// An OS lease on a worktree's outputs, independent of repository coordination.
80/// Kept by the process cleanup owner until descendants terminate, including
81/// reparented tools retaining the inherited liveness witness.
82#[derive(Debug)]
83pub(crate) struct OutputLease {
84    _file: File,
85    _activity: bool,
86}
87
88impl OutputLease {
89    #[cfg(unix)]
90    pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
91        use std::os::fd::AsRawFd;
92        self._file.as_raw_fd()
93    }
94
95    pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
96        if !worktree.join(".git").is_file() {
97            return Ok(None);
98        }
99        Self::take(worktree, false).map(Some)
100    }
101    pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
102        use std::hash::{Hash, Hasher};
103        let mut hash = std::collections::hash_map::DefaultHasher::new();
104        let mut normalized = cache.to_owned();
105        let mut missing = Vec::new();
106        while !normalized.exists() {
107            let Some(name) = normalized.file_name() else {
108                break;
109            };
110            missing.push(name.to_owned());
111            if !normalized.pop() {
112                break;
113            }
114        }
115        normalized = normalized.canonicalize().unwrap_or(normalized);
116        for part in missing.into_iter().rev() {
117            normalized.push(part);
118        }
119        normalized.hash(&mut hash);
120        let dir = home.join("cache-leases");
121        std::fs::create_dir_all(&dir)?;
122        Self::lock(
123            &dir.join(format!("outputs-{:x}.lock", hash.finish())),
124            exclusive,
125        )
126    }
127    /// Reserve headroom once per volume while admitting a managed writer.
128    /// These are estimates, not filesystem quotas. The ordinary emergency
129    /// monitor remains necessary for writes exceeding the reservation.
130    #[cfg(unix)]
131    pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
132        Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
133    }
134
135    #[cfg(unix)]
136    pub(crate) fn admit_with(
137        home: &Path,
138        paths: &[PathBuf],
139        min_free: u64,
140        free_bytes: impl Fn(&Path) -> Result<u64>,
141    ) -> Result<Vec<Self>> {
142        use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
143        if min_free == 0 {
144            return Ok(Vec::new());
145        }
146        const RESERVE: u64 = 1024 * 1024 * 1024;
147        std::fs::create_dir_all(home)?;
148        let dir = home.canonicalize()?.join("disk-admission");
149        std::fs::create_dir_all(&dir)?;
150        ensure!(
151            plain_dir(&dir) && dir.canonicalize()? == dir,
152            "disk admission directory is not canonical"
153        );
154        let mut devices = std::collections::BTreeSet::new();
155        let mut reservations = Vec::new();
156        for path in paths {
157            let probe = path
158                .ancestors()
159                .find(|p| p.exists())
160                .context("disk admission path missing")?;
161            let device = std::fs::metadata(probe)?.dev();
162            if !devices.insert(device) {
163                continue;
164            }
165            let open = |path: &Path| -> Result<File> {
166                Ok(OpenOptions::new()
167                    .read(true)
168                    .write(true)
169                    .create(true)
170                    .truncate(false)
171                    .custom_flags(libc::O_NOFOLLOW)
172                    .open(path)?)
173            };
174            // Explicitly unlock on return. A concurrent fork can briefly
175            // inherit this descriptor before exec closes it; closing only the
176            // parent's copy would leave the namespace spuriously busy.
177            let admission = Self {
178                _file: open(&dir.join(format!("{device}.lock")))?,
179                _activity: false,
180            };
181            let started = std::time::Instant::now();
182            loop {
183                match crate::pr_head::try_lock_file(&admission._file) {
184                    Ok(()) => break,
185                    Err(error)
186                        if matches!(
187                            error.kind(),
188                            std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
189                        ) && started.elapsed() < std::time::Duration::from_secs(1) =>
190                    {
191                        std::thread::sleep(std::time::Duration::from_millis(5));
192                    }
193                    Err(error) => {
194                        return Err(error)
195                            .context("disk admission busy or unavailable; retry without dispatch");
196                    }
197                }
198            }
199            let mut active = 0u64;
200            let mut available = None;
201            for slot in 0..1024 {
202                let file = dir.join(format!("{device}-{slot}.reserve"));
203                if !file.exists() {
204                    if available.is_none() {
205                        available = Some(file);
206                    }
207                    continue;
208                }
209                let file = open(&file)?;
210                match crate::pr_head::try_lock_file(&file) {
211                    Ok(()) if available.is_none() => {
212                        available = Some(dir.join(format!("{device}-{slot}.reserve")))
213                    }
214                    Ok(()) => {}
215                    Err(_) => active = active.saturating_add(RESERVE),
216                }
217            }
218            let free = free_bytes(probe)?;
219            let required = min_free.saturating_add(active).saturating_add(RESERVE);
220            ensure!(
221                free >= required,
222                "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
223            );
224            let file = open(&available.context("disk admission reservation slots exhausted")?)?;
225            crate::pr_head::try_lock_file(&file)?;
226            reservations.push(Self {
227                _file: file,
228                _activity: false,
229            });
230        }
231        Ok(reservations)
232    }
233
234    #[cfg(not(unix))]
235    pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
236        // Registered seats already fail closed without Unix containment.
237        Ok(Vec::new())
238    }
239
240    pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
241        let out = std::process::Command::new("git")
242            .args(["rev-parse", "--absolute-git-dir"])
243            .current_dir(worktree)
244            .quiet()
245            .output()?;
246        ensure!(
247            out.status.success(),
248            "cannot identify output lease directory"
249        );
250        let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
251        Self::lock(&dir.join("magi-output.lock"), exclusive)
252    }
253    fn lock(path: &Path, exclusive: bool) -> Result<Self> {
254        let file = OpenOptions::new()
255            .create(true)
256            .truncate(false)
257            .read(true)
258            .write(true)
259            .open(path)
260            .with_context(|| format!("output lease: open {}", path.display()))?;
261        #[cfg(unix)]
262        {
263            use std::os::fd::AsRawFd;
264            let mode = if exclusive {
265                libc::LOCK_EX
266            } else {
267                libc::LOCK_SH
268            };
269            // SAFETY: live file descriptor, no retained pointers.
270            ensure!(
271                unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
272                "output lease: outputs are in use or being reclaimed: {}",
273                path.display()
274            );
275        }
276        #[cfg(not(unix))]
277        let _ = exclusive;
278        if !exclusive {
279            // Empty legacy/maintenance-only leases carry no activity witness.
280            file.set_len(1)?;
281            file.set_modified(std::time::SystemTime::now())
282                .with_context(|| format!("record output activity {}", path.display()))?;
283        }
284        Ok(Self {
285            _file: file,
286            _activity: !exclusive,
287        })
288    }
289}
290#[cfg(unix)]
291impl Drop for OutputLease {
292    fn drop(&mut self) {
293        use std::os::fd::AsRawFd;
294        if self._activity
295            && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
296        {
297            // If release time cannot be recorded, retain a nonempty invalid
298            // witness rather than trusting the old acquisition timestamp.
299            let _ = self._file.set_len(2);
300            tracing::warn!("could not record output lease release activity: {error}");
301        }
302        // SAFETY: unlock this guard's live descriptor, including inherited copies.
303        unsafe {
304            libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
305        }
306    }
307}
308
309#[derive(Default, Debug, Serialize, Deserialize)]
310#[serde(default)]
311pub(crate) struct Report {
312    pub checked_at: Option<Timestamp>,
313    pub limit_bytes: u64,
314    pub reclaim_budget_bytes: u64,
315    /// Logical inventory, including protected and unproven data.
316    pub measured_bytes: u64,
317    /// Logical bytes unlinked, not a measurement of physical disk recovery.
318    pub freed_bytes: u64,
319    pub measured_allocated_file_bytes: Option<u64>,
320    /// Candidates proven under locks in this pass; uninspected data is excluded.
321    pub proven_reclaimable_bytes: u64,
322    /// Estimated allocated blocks recoverable if every proven link is removed.
323    /// Inodes with links outside the candidate set contribute zero.
324    pub proven_reclaimable_allocated_bytes: Option<u64>,
325    pub protected_or_unproven_bytes: u64,
326    /// Deduplicated allocated file blocks whose final link was unlinked. Open
327    /// descriptors and shared extents can prevent an equivalent physical gain.
328    pub estimated_released_allocated_bytes: Option<u64>,
329    /// Observed free-space change on the home volume; concurrent writes, clones
330    /// and deferred reclamation mean this is not attributable to this sweep.
331    pub observed_home_free_change_bytes: Option<i64>,
332    pub retired: usize,
333    pub evicted_copies: usize,
334    pub removed_empty_roots: usize,
335    pub source_unlinked_logical_bytes: u64,
336    pub source_unlinked_allocated_bytes: Option<u64>,
337    pub source_evidence_added_logical_bytes: u64,
338    pub source_evidence_added_allocated_bytes: Option<u64>,
339    pub retained: Vec<String>,
340    pub failures: Vec<String>,
341}
342
343#[cfg(unix)]
344fn allocated_inventory(
345    root: &Path,
346    seen: &mut std::collections::BTreeSet<(u64, u64)>,
347) -> Result<u64> {
348    use std::os::unix::fs::MetadataExt;
349    ensure!(
350        plain_dir(root) && root.canonicalize()? == root,
351        "allocated inventory root is not canonical"
352    );
353    let mut dirs = vec![root.to_owned()];
354    let mut bytes = 0u64;
355    while let Some(dir) = dirs.pop() {
356        for entry in std::fs::read_dir(dir)? {
357            let path = entry?.path();
358            let metadata = std::fs::symlink_metadata(&path)?;
359            if metadata.is_dir() {
360                dirs.push(path);
361            } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
362                bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
363            }
364        }
365    }
366    Ok(bytes)
367}
368
369#[derive(Default)]
370struct ReleasedAllocation {
371    #[cfg(unix)]
372    files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
373    #[cfg(unix)]
374    removed: std::collections::BTreeMap<(u64, u64), u64>,
375}
376
377impl ReleasedAllocation {
378    fn of(paths: &[PathBuf]) -> Result<Self> {
379        #[cfg(unix)]
380        {
381            use std::os::unix::fs::MetadataExt;
382            let mut files = std::collections::BTreeMap::new();
383            for path in paths {
384                let m = std::fs::symlink_metadata(path)?;
385                ensure!(m.is_file(), "allocation candidate is not regular");
386                files.insert(
387                    path.clone(),
388                    (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
389                );
390            }
391            Ok(Self {
392                files,
393                removed: Default::default(),
394            })
395        }
396        #[cfg(not(unix))]
397        {
398            let _ = paths;
399            Ok(Self::default())
400        }
401    }
402    fn reclaimable_allocated_bytes(&self) -> Option<u64> {
403        #[cfg(unix)]
404        {
405            let mut inodes = std::collections::BTreeMap::new();
406            for &(device, inode, links, blocks) in self.files.values() {
407                let entry = inodes
408                    .entry((device, inode))
409                    .or_insert((0u64, links, blocks));
410                entry.0 += 1;
411            }
412            Some(
413                inodes
414                    .values()
415                    .filter(|(count, links, _)| count == links)
416                    .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
417            )
418        }
419        #[cfg(not(unix))]
420        {
421            None
422        }
423    }
424
425    fn unlinked(&mut self, path: &Path) -> Option<u64> {
426        #[cfg(unix)]
427        {
428            let (device, inode, links, blocks) = self.files[path];
429            let removed = self.removed.entry((device, inode)).or_default();
430            *removed += 1;
431            Some(if *removed == links { blocks } else { 0 })
432        }
433        #[cfg(not(unix))]
434        {
435            let _ = path;
436            None
437        }
438    }
439}
440
441fn plain_dir(path: &Path) -> bool {
442    std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
443}
444
445/// Old run records outlive their checkout and output directories. Only an
446/// absent path means nothing to reclaim; inaccessible paths remain errors.
447fn run_paths_missing(state: &RunState) -> Result<bool> {
448    for path in [state.worktree_root(), state.repo.clone()] {
449        match std::fs::metadata(&path) {
450            Ok(_) => {}
451            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
452            Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
453        }
454    }
455    Ok(false)
456}
457
458/// Enumerate actual registered worktrees, not just candidates in run.json.
459async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
460    let root = state.worktree_root();
461    if !plain_dir(&root) {
462        return Ok(Vec::new());
463    }
464    let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
465    registered_worktrees(state, &list)
466}
467
468fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
469    let root = state.worktree_root();
470    if !plain_dir(&root) {
471        return Ok(Vec::new());
472    }
473    let root = root.canonicalize()?;
474    let mut paths = Vec::new();
475    for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
476        let path = PathBuf::from(path);
477        #[cfg(windows)]
478        let path = if plain_dir(&path) {
479            git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
480        } else {
481            continue;
482        };
483        if plain_dir(&path)
484            && path.parent() == Some(root.as_path())
485            && path.canonicalize()? == path
486            && path.join(".git").is_file()
487        {
488            paths.push(path);
489        }
490    }
491    Ok(paths)
492}
493
494/// Recognize Cargo's generated directory, never merely an ignored directory.
495/// Unknown root entries are retained; cache identity/profile paths fail closed.
496async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
497    validate_cargo_target(worktree, false).await
498}
499
500async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
501    let target = worktree.join("target");
502    ensure!(plain_dir(&target), "target absent or not a plain directory");
503    ensure!(target.canonicalize()? == target, "target path changed");
504    ensure!(
505        git::git(worktree, &["ls-files", "--", "target"])
506            .await?
507            .is_empty(),
508        "tracked target contents"
509    );
510    ensure!(
511        git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
512            .await?
513            .ok(),
514        "target is not ignored"
515    );
516    cache_tag(&target, allow_missing_tag)?;
517    let path = target.join(".rustc_info.json");
518    plain_file(&path, "rustc identity")?;
519    let info: serde_json::Value = serde_json::from_slice(
520        &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
521    )
522    .with_context(|| format!("parse rustc identity {}", path.display()))?;
523    ensure!(
524        info.get("rustc_fingerprint").is_some(),
525        "unknown Cargo identity {}",
526        path.display()
527    );
528    if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
529        ensure!(
530            info.get("rustc_fingerprint")
531                .is_some_and(serde_json::Value::is_u64)
532                && info
533                    .get("outputs")
534                    .and_then(serde_json::Value::as_object)
535                    .is_some_and(|outputs| outputs.values().any(|output| {
536                        output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
537                            && output
538                                .get("stdout")
539                                .and_then(serde_json::Value::as_str)
540                                .is_some_and(|stdout| stdout.starts_with("rustc "))
541                            && output
542                                .get("stderr")
543                                .is_some_and(serde_json::Value::is_string)
544                    }))
545                && info
546                    .get("successes")
547                    .is_some_and(serde_json::Value::is_object),
548            "missing-tag rustc identity structure invalid: {}",
549            path.display()
550        );
551        let mut profiles = 0;
552        for profile in ["debug", "release"] {
553            let dir = target.join(profile);
554            if !dir.exists() {
555                continue;
556            }
557            for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
558                ensure!(
559                    plain_dir(&subdir) && subdir.canonicalize()? == subdir,
560                    "missing-tag profile proof: {} is not a plain directory",
561                    subdir.display()
562                );
563            }
564            plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
565            profiles += 1;
566        }
567        ensure!(
568            profiles > 0,
569            "missing-tag proof: no Cargo profile in {}",
570            target.display()
571        );
572    }
573    for name in ["debug", "release"] {
574        let path = target.join(name);
575        match std::fs::symlink_metadata(&path) {
576            Ok(metadata) => ensure!(
577                metadata.is_dir() && path.canonicalize()? == path,
578                "profile validation: not a plain directory: {}",
579                path.display()
580            ),
581            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
582            Err(error) => {
583                return Err(error)
584                    .with_context(|| format!("profile validation: inspect {}", path.display()));
585            }
586        }
587    }
588    Ok(target)
589}
590
591fn plain_file(path: &Path, stage: &str) -> Result<()> {
592    let meta = std::fs::symlink_metadata(path)
593        .with_context(|| format!("{stage}: inspect {}", path.display()))?;
594    ensure!(
595        meta.is_file() && path.canonicalize()? == path,
596        "{stage}: not a plain file: {}",
597        path.display()
598    );
599    Ok(())
600}
601
602fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
603    let path = target.join("CACHEDIR.TAG");
604    match std::fs::symlink_metadata(&path) {
605        Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
606            return Ok(());
607        }
608        result => {
609            result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
610        }
611    }
612    plain_file(&path, "cache tag")?;
613    let tag = std::fs::read_to_string(&path)
614        .with_context(|| format!("cache tag: read {}", path.display()))?;
615    ensure!(
616        tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
617        "unknown Cargo cache tag: {}",
618        path.display()
619    );
620    Ok(())
621}
622
623/// Observe compiler metadata and artifacts under the output and Cargo locks.
624/// Unknown contents never become deletion authority. Unreadable timestamps or
625/// a future timestamp retain outputs, even when the configured grace is zero.
626fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
627    let mut dirs = vec![target.to_owned()];
628    let mut entries = 0;
629    while let Some(dir) = dirs.pop() {
630        for entry in std::fs::read_dir(&dir)
631            .with_context(|| format!("output activity: list {}", dir.display()))?
632        {
633            let path = entry?.path();
634            entries += 1;
635            ensure!(
636                entries <= 100_000,
637                "output activity inventory too large: {}",
638                target.display()
639            );
640            let metadata = std::fs::symlink_metadata(&path)
641                .with_context(|| format!("output activity: inspect {}", path.display()))?;
642            if metadata.is_dir() {
643                // Only supported Cargo profiles supply execution evidence.
644                if dir != target
645                    || matches!(
646                        path.file_name().and_then(|s| s.to_str()),
647                        Some("debug" | "release")
648                    )
649                {
650                    dirs.push(path);
651                }
652            } else if metadata.is_file()
653                && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
654            {
655                let at =
656                    Timestamp::try_from(metadata.modified().with_context(|| {
657                        format!("output activity: modified {}", path.display())
658                    })?)?;
659                ensure!(
660                    crate::clean::due(now, at, grace),
661                    "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
662                    path.display()
663                );
664            }
665        }
666    }
667    Ok(())
668}
669
670pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
671    use std::io::Read as _;
672    let mut header = [0; 8];
673    let n = File::open(path)?.read(&mut header)?;
674    let header = &header[..n];
675    Ok(header.starts_with(b"\x7fELF")
676        || header.starts_with(b"!<arch>\n")
677        || header.starts_with(b"rust\0\0\0")
678        || header.starts_with(b"MZ\x90\0")
679        || [
680            [0xfe, 0xed, 0xfa, 0xce],
681            [0xce, 0xfa, 0xed, 0xfe],
682            [0xfe, 0xed, 0xfa, 0xcf],
683            [0xcf, 0xfa, 0xed, 0xfe],
684            [0xca, 0xfe, 0xba, 0xbe],
685            [0xca, 0xfe, 0xba, 0xbf],
686        ]
687        .iter()
688        .any(|magic| header.starts_with(magic)))
689}
690
691/// Legacy tags identify a cache, not ownership of every file inside it.
692/// Only direct rustc outputs named by dep-info may be reclaimed. Build-script
693/// output, incremental caches, unknown files and the manifests themselves stay.
694fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
695    let mut outputs = std::collections::BTreeSet::new();
696    for profile in ["debug", "release"] {
697        for dir in [target.join(profile), target.join(profile).join("deps")] {
698            if !plain_dir(&dir) || dir.canonicalize()? != dir {
699                continue;
700            }
701            for entry in std::fs::read_dir(&dir)? {
702                let path = entry?.path();
703                if path.extension().is_none_or(|ext| ext != "d")
704                    || !std::fs::symlink_metadata(&path)?.is_file()
705                    || std::fs::metadata(&path)?.len() > 1024 * 1024
706                    || path.canonicalize()? != path
707                {
708                    continue;
709                }
710                let body = std::fs::read_to_string(&path)?;
711                for line in body.lines() {
712                    let Some((names, dependencies)) = line.split_once(": ") else {
713                        continue;
714                    };
715                    if dependencies.trim().is_empty() {
716                        continue;
717                    }
718                    // Rustc escapes spaces and backslashes in Make paths.
719                    // Unsupported Make syntax fails closed for that record.
720                    let mut words = Vec::new();
721                    let mut word = String::new();
722                    let mut escaped = false;
723                    for c in names.chars() {
724                        if escaped {
725                            word.push(c);
726                            escaped = false;
727                        } else if c == '\\' {
728                            escaped = true;
729                        } else if c.is_whitespace() {
730                            if !word.is_empty() {
731                                words.push(std::mem::take(&mut word));
732                            }
733                        } else {
734                            word.push(c);
735                        }
736                    }
737                    if escaped || word.contains(['$', '%']) {
738                        continue;
739                    }
740                    if !word.is_empty() {
741                        words.push(word);
742                    }
743                    for name in words {
744                        let output = PathBuf::from(name);
745                        let stem = path.file_stem().unwrap_or_default().to_string_lossy();
746                        let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
747                        let artifact_name =
748                            output_stem == stem || output_stem == format!("lib{stem}");
749                        let artifact_kind = output.extension().is_none_or(|ext| {
750                            matches!(
751                                ext.to_str(),
752                                Some(
753                                    "rlib"
754                                        | "rmeta"
755                                        | "so"
756                                        | "dylib"
757                                        | "a"
758                                        | "lib"
759                                        | "dll"
760                                        | "exe"
761                                        | "pdb"
762                                )
763                            )
764                        });
765                        if artifact_name
766                            && artifact_kind
767                            && output.parent() == Some(dir.as_path())
768                            && output.extension().is_none_or(|ext| ext != "d")
769                            && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
770                            && output.canonicalize().is_ok_and(|p| p == output)
771                            && artifact_signature(&output)?
772                        {
773                            outputs.insert(output);
774                        }
775                    }
776                }
777            }
778        }
779    }
780    Ok(outputs.into_iter().collect())
781}
782
783#[derive(Serialize, Deserialize)]
784struct CargoResidue {
785    target: PathBuf,
786    files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
787}
788
789// Keep the exact metadata as historical evidence. An old proof cannot adopt
790// newly added files, changed metadata, symlinks or a different worktree.
791fn residue_files(
792    target: &Path,
793    outputs: &[PathBuf],
794) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
795    let mut files = std::collections::BTreeMap::new();
796    let mut dirs = vec![target.to_owned()];
797    let mut bytes = 0usize;
798    let mut entries = 0usize;
799    while let Some(dir) = dirs.pop() {
800        for entry in std::fs::read_dir(dir)? {
801            let path = entry?.path();
802            entries += 1;
803            ensure!(entries <= 4096, "residue inventory too large");
804            let metadata = std::fs::symlink_metadata(&path)?;
805            ensure!(
806                !metadata.file_type().is_symlink(),
807                "symlink in Cargo residue"
808            );
809            if metadata.is_dir() {
810                dirs.push(path);
811            } else {
812                ensure!(metadata.is_file(), "non-file in Cargo residue");
813                if outputs.contains(&path) {
814                    continue;
815                }
816                ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
817                let body = std::fs::read(&path)?;
818                bytes = bytes.saturating_add(body.len());
819                ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
820                files.insert(path.strip_prefix(target)?.to_owned(), body);
821            }
822        }
823    }
824    Ok(files)
825}
826
827fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
828    Ok(home
829        .join("runs")
830        .join(id)
831        .join("build-residue")
832        .join(worktree.file_name().context("missing worktree name")?))
833}
834
835// Artifact eviction tolerates unrelated root data. Whole-target removal must
836// still reject it, including empty directories absent from the file inventory.
837fn residue_root(target: &Path) -> Result<()> {
838    for entry in std::fs::read_dir(target)? {
839        let name = entry?.file_name();
840        ensure!(
841            matches!(
842                name.to_str(),
843                Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
844            ),
845            "unrecognized Cargo residue root entry"
846        );
847    }
848    Ok(())
849}
850
851fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
852    cache_tag(target, false)?;
853    residue_root(target)?;
854    let files = residue_files(target, outputs)?;
855    for (relative, body) in &files {
856        let path = target.join(relative);
857        let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
858        let lock = matches!(
859            relative.to_str(),
860            Some("debug/.cargo-lock" | "release/.cargo-lock")
861        ) && body.is_empty();
862        let dep_info = path.extension().is_some_and(|ext| ext == "d")
863            && outputs.iter().any(|output| {
864                output.parent() == path.parent()
865                    && (output.file_stem() == path.file_stem()
866                        || output.file_stem().is_some_and(|stem| {
867                            stem.to_string_lossy()
868                                == format!(
869                                    "lib{}",
870                                    path.file_stem().unwrap_or_default().to_string_lossy()
871                                )
872                        }))
873            });
874        ensure!(identity || lock || dep_info, "unproven Cargo residue");
875    }
876    let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
877    std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
878    let tmp = proof.with_extension("tmp");
879    std::fs::write(
880        &tmp,
881        serde_json::to_vec(&CargoResidue {
882            target: target.to_owned(),
883            files,
884        })?,
885    )?;
886    std::fs::rename(tmp, proof)?;
887    Ok(())
888}
889
890pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
891    let target = cargo_target(worktree).await?;
892    residue_root(&target)?;
893    let locks = cargo_locks(&target)?;
894    let proof: CargoResidue =
895        serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
896    ensure!(
897        proof.target == target && proof.files == residue_files(&target, &[])?,
898        "Cargo residue changed or contains unproven files"
899    );
900    Ok(locks)
901}
902
903/// Supplemental exclusion for Cargo builds launched outside the managed seats.
904pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
905    let mut held = Vec::new();
906    for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
907        [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
908    }) {
909        let lock = target.join(profile).join(name);
910        match std::fs::symlink_metadata(&lock) {
911            Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
912            result => {
913                result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
914            }
915        }
916        {
917            plain_file(&lock, "Cargo lock")?;
918            let file = OpenOptions::new()
919                .read(true)
920                .write(true)
921                .open(&lock)
922                .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
923            crate::pr_head::try_lock_file(&file)
924                .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
925            held.push(file);
926        }
927    }
928    Ok(held)
929}
930
931/// All repositories share one cap and one per-pass deletion budget. The report
932/// survives a restart and explains every skipped run/target or failed operation.
933pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
934    let home = home.to_owned();
935    let disk = disk.clone();
936    let runtime = tokio::runtime::Handle::current();
937    tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
938        .await
939        .unwrap_or_else(|e| Report {
940            failures: vec![format!("retention worker failed: {e}")],
941            ..Report::default()
942        })
943}
944
945#[derive(Default, Serialize, Deserialize)]
946struct Measurement {
947    at: Option<Timestamp>,
948    bytes: u64,
949}
950
951async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
952    let mut report = Report {
953        checked_at: Some(now),
954        limit_bytes: disk.worktree_limit_bytes,
955        reclaim_budget_bytes: disk.worktree_reclaim_bytes,
956        ..Report::default()
957    };
958    if !cfg!(unix) {
959        report
960            .failures
961            .push("Automatic output reclamation requires Unix process containment".into());
962        return report;
963    }
964    let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
965        return report;
966    };
967    let mut targets = Vec::new();
968    let mut under_pressure = false;
969    let mut registrations = std::collections::HashMap::new();
970    let measurements_path = home.join("output-measurements.json");
971    let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
972        std::fs::read(&measurements_path)
973            .ok()
974            .and_then(|b| serde_json::from_slice(&b).ok())
975            .unwrap_or_default();
976    let mut observed = std::collections::BTreeSet::new();
977    let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
978        .into_iter()
979        .flatten()
980        .flatten()
981        .map(|e| e.file_name().to_string_lossy().into_owned())
982        .collect();
983    ids.sort();
984    for id in ids {
985        if !crate::run::is_run_id(&id) {
986            continue;
987        }
988        let state = match RunState::load_under(&id, home) {
989            Ok(state) => state,
990            Err(e) => {
991                report
992                    .retained
993                    .push(format!("{id}: unreadable run ownership: {e:#}"));
994                continue;
995            }
996        };
997        if disk.worktree_limit_bytes == 0 {
998            continue;
999        }
1000        match run_paths_missing(&state) {
1001            Ok(true) => continue,
1002            Ok(false) => {}
1003            Err(e) => {
1004                report.failures.push(format!("{id}: {e:#}"));
1005                continue;
1006            }
1007        }
1008        let list = match registrations.entry(state.repo.clone()) {
1009            std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1010            std::collections::hash_map::Entry::Vacant(entry) => {
1011                match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1012                    Ok(list) => entry.insert(list),
1013                    Err(e) => {
1014                        // The checkout may disappear after the inventory check.
1015                        // Do not confuse a missing Git executable with that case.
1016                        if matches!(run_paths_missing(&state), Ok(true)) {
1017                            continue;
1018                        }
1019                        report.failures.push(format!("{id}: {e:#}"));
1020                        continue;
1021                    }
1022                }
1023            }
1024        };
1025        let paths = match registered_worktrees(&state, list) {
1026            Ok(paths) => paths,
1027            Err(e) => {
1028                report.failures.push(format!("{id}: {e:#}"));
1029                continue;
1030            }
1031        };
1032        for path in paths {
1033            if !path.join("target").exists() {
1034                continue;
1035            }
1036            let target = path.join("target");
1037            observed.insert(target.clone());
1038            let pressure =
1039                crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1040            under_pressure |= pressure;
1041            let measurement = measurements.entry(target).or_default();
1042            if pressure
1043                || measurement
1044                    .at
1045                    .is_none_or(|at| crate::clean::due(now, at, 300))
1046            {
1047                measurement.bytes = crate::disk::dir_size(&path.join("target"));
1048                measurement.at = Some(now);
1049            }
1050            let size = measurement.bytes;
1051            report.measured_bytes = report.measured_bytes.saturating_add(size);
1052            targets.push((state.output_activity(), id.clone(), path, size));
1053        }
1054    }
1055    if under_pressure {
1056        // Inventory sizes prioritize scarce-space recovery only; deletion authority
1057        // and budget accounting come from declared outputs measured under locks.
1058        targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1059    } else {
1060        targets.sort_by_key(|t| t.0);
1061    }
1062    let free_before = crate::disk::free_bytes(home).ok();
1063    #[cfg(unix)]
1064    {
1065        let mut seen = std::collections::BTreeSet::new();
1066        report.measured_allocated_file_bytes = Some(0);
1067        report.estimated_released_allocated_bytes = Some(0);
1068        report.proven_reclaimable_allocated_bytes = Some(0);
1069        for (_, _, path, _) in &targets {
1070            report.measured_allocated_file_bytes = report
1071                .measured_allocated_file_bytes
1072                .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1073                .map(|(a, b)| a.saturating_add(b));
1074        }
1075    }
1076    let mut remaining = report.measured_bytes;
1077    let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1078    let mut deletion_started = false;
1079    let mut oversized_started = false;
1080    for (updated, id, path, _) in targets {
1081        if disk.worktree_limit_bytes == 0
1082            || remaining <= disk.worktree_limit_bytes
1083                && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1084        {
1085            report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1086            continue;
1087        }
1088        if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1089            report
1090                .retained
1091                .push(format!("{id}: per-pass byte budget spent"));
1092            continue;
1093        }
1094        let result = async {
1095            ensure!(
1096                crate::clean::due(now, updated, disk.fold_grace_secs),
1097                "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1098            );
1099            let _owner = crate::pr_head::owner_lock(home, &id)?;
1100            let state = RunState::load_under(&id, home)?;
1101            ensure!(
1102                state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1103                    == Liveness::Dead,
1104                "run live or ownership unknown"
1105            );
1106            ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1107            let _driver = crate::driver::maintenance(home, &id)?;
1108            let _outputs = OutputLease::take(&path, true)?;
1109            ensure!(
1110                worktrees(&state).await?.contains(&path),
1111                "worktree registration changed"
1112            );
1113            let target = validate_cargo_target(&path, true).await
1114                .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1115            let _cargo = cargo_locks(&target)
1116                .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1117            // Cargo may have finished between identity inspection and locking.
1118            // Revalidate the same authority with all writers excluded.
1119            validate_cargo_target(&path, true).await
1120                .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1121            let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1122            if lease_meta.len() > 0 {
1123                ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1124                let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1125                ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1126                    "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1127            }
1128            output_age(&target, now, disk.fold_grace_secs)
1129                .with_context(|| format!("stage output-age: {}", target.display()))?;
1130            let mut outputs = declared_outputs(&target)
1131                .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1132            let incremental = match crate::incremental::candidates(&path) {
1133                Ok(candidates) => candidates,
1134                Err(error) => {
1135                    report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1136                    crate::incremental::Reclaim::default()
1137                }
1138            };
1139            let incremental_files = incremental.files.clone();
1140            outputs.extend(incremental_files.iter().cloned());
1141            outputs.sort();
1142            outputs.dedup();
1143            ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1144            let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1145                .with_context(|| format!("stage measure: {}", p.display())))
1146                .collect::<Result<Vec<_>>>()?;
1147            let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1148            report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1149            let mut allocation = ReleasedAllocation::of(&outputs)?;
1150            report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1151                .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1152            let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1153            // 予算は1回の削除量を抑えるもので、大きな出力を永久に保持するためではない。
1154            // 正の予算では最初の削除対象だけ超過を許し、その後は予算消費済みとする。
1155            // 途中で失敗しても再度許可しないよう、例外は削除開始時に消費する。
1156            ensure!(!deletion_started || !exceeds_budget,
1157                "declared outputs exceed remaining per-pass byte budget");
1158            // Record adoption before deletion; interrupted deletion is revalidated
1159            // on the next pass. No source file or branch is modified.
1160            let ledger = home.join("runs").join(&id).join("build-outputs.json");
1161            std::fs::write(
1162                &ledger,
1163                serde_json::to_vec(
1164                    &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1165                )?,
1166            ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1167            // Unknown leftovers still permit artifact-only eviction, but never
1168            // authorize retirement. Only a complete proven residue is adopted.
1169            if !incremental_files.is_empty() {
1170                report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1171            } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1172                report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1173            }
1174            ensure!(worktrees(&state).await?.contains(&path),
1175                "worktree registration changed before deletion");
1176            deletion_started = true;
1177            oversized_started = exceeds_budget;
1178            measurements.remove(&target);
1179            for (output, size) in outputs.iter().zip(sizes) {
1180                std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1181                report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1182                    .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1183                // Preserve actual progress even if a later removal fails.
1184                report.freed_bytes = report.freed_bytes.saturating_add(size);
1185                remaining = remaining.saturating_sub(size);
1186            }
1187            incremental.finish().context("finish incremental reclamation")?;
1188            report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1189            Ok::<_, anyhow::Error>(())
1190        }
1191        .await;
1192        if let Err(e) = result {
1193            report.retained.push(format!(
1194                "{id}/{}: {e:#}",
1195                path.file_name().unwrap_or_default().to_string_lossy()
1196            ));
1197        }
1198    }
1199    measurements.retain(|path, _| observed.contains(path));
1200    if let Ok(body) = serde_json::to_vec(&measurements)
1201        && let Err(e) = std::fs::write(&measurements_path, body)
1202    {
1203        report.failures.push(format!("output measurements: {e}"));
1204    }
1205    copies::evict(home, disk, now, &mut report).await;
1206    retire_empty(home, disk, now, &mut report).await;
1207    report.protected_or_unproven_bytes = report
1208        .measured_bytes
1209        .saturating_sub(report.proven_reclaimable_bytes);
1210    let pressure_remaining = pressure_paths
1211        .iter()
1212        .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1213    if !report.failures.is_empty()
1214        || disk.worktree_limit_bytes > 0
1215            && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1216    {
1217        let details = report
1218            .retained
1219            .iter()
1220            .filter(|reason| {
1221                !reason.contains("within policy")
1222                    && !reason.ends_with("unmanifested files and Cargo metadata retained")
1223            })
1224            .take(4)
1225            .chain(report.failures.iter().take(2))
1226            .cloned()
1227            .collect::<Vec<_>>()
1228            .join("; ");
1229        crate::notices::raise_in(
1230            home,
1231            crate::notices::Notice::warn(
1232                "disk:retention",
1233                format!(
1234                    "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1235                    report.measured_bytes,
1236                    report.proven_reclaimable_bytes,
1237                    report.protected_or_unproven_bytes,
1238                    report.freed_bytes,
1239                    if report.proven_reclaimable_bytes == 0 {
1240                        "No inspected contents have deletion authority."
1241                    } else {
1242                        "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1243                    }
1244                ),
1245            ),
1246        );
1247    }
1248    report.observed_home_free_change_bytes = free_before
1249        .zip(crate::disk::free_bytes(home).ok())
1250        .map(|(before, after)| {
1251            (i128::from(after) - i128::from(before))
1252                .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1253        });
1254    let store = home.join("retention.json");
1255    if let Ok(body) = serde_json::to_vec_pretty(&report) {
1256        let tmp = home.join("retention.json.tmp");
1257        if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1258            tracing::warn!("retention report could not be saved: {e}");
1259        }
1260    }
1261    tracing::info!(
1262        measured_bytes = report.measured_bytes,
1263        freed_bytes = report.freed_bytes,
1264        retired = report.retired,
1265        retained = report.retained.len(),
1266        failures = report.failures.len(),
1267        "retained worktree cleanup"
1268    );
1269    for reason in report.retained.iter().chain(&report.failures) {
1270        tracing::debug!(%reason, "worktree retained");
1271    }
1272    report
1273}
1274
1275async fn retire_empty(
1276    home: &Path,
1277    disk: &crate::config::Disk,
1278    now: Timestamp,
1279    report: &mut Report,
1280) {
1281    if !disk.auto_fold {
1282        return;
1283    }
1284    let queue = crate::queue::Queue::at(home.join("queue"));
1285    let questions = crate::ask::Questions::at(home.join("questions"));
1286    let open = questions.list();
1287    for listed in queue.list() {
1288        let Ok(_claim) = queue.claim(&listed.id) else {
1289            continue;
1290        };
1291        let Ok(task) = queue.get(&listed.id) else {
1292            continue;
1293        };
1294        if open
1295            .iter()
1296            .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1297        {
1298            report.retained.push(format!(
1299                "{}: task has an unanswered owner decision",
1300                task.id
1301            ));
1302            continue;
1303        }
1304        let sources: Vec<_> = task
1305            .runs
1306            .iter()
1307            .collect::<std::collections::BTreeSet<_>>()
1308            .into_iter()
1309            .cloned()
1310            .collect();
1311        let mut useful = Vec::new();
1312        let mut selected = Vec::new();
1313        let mut public = Vec::new();
1314        let mut unreadable = false;
1315        for id in &sources {
1316            let Ok(state) = RunState::load_under(id, home) else {
1317                unreadable = true;
1318                continue;
1319            };
1320            if state.released() || state.retired_to.is_some() {
1321                continue;
1322            }
1323            if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1324                selected.push(state.clone());
1325            }
1326            if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1327                public.push(state.clone());
1328            }
1329            if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1330                useful.push(state);
1331            }
1332        }
1333        // Prefer a durable owner-approved selection, then one existing PR
1334        // owner, then an unambiguous useful source. Distinct historical changes
1335        // without an owner selection cannot be resolved by creation time.
1336        let pool = if !selected.is_empty() {
1337            selected
1338        } else if !public.is_empty() {
1339            public
1340        } else {
1341            useful
1342        };
1343        if unreadable || pool.len() != 1 {
1344            report.retained.push(format!(
1345                "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1346                task.id,
1347                sources.len()
1348            ));
1349            continue;
1350        }
1351        let authority = &pool[0];
1352        if authority.candidates.is_empty() {
1353            continue;
1354        }
1355        let latest = &authority.id;
1356        for id in &sources {
1357            if id == latest || report.retired >= 5 {
1358                continue;
1359            }
1360            if let Ok(state) = RunState::load_under(id, home)
1361                && state.retired_to.as_ref() == Some(latest)
1362                && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1363            {
1364                continue;
1365            }
1366            let result = async {
1367                ensure!(
1368                    !task
1369                        .resume_override
1370                        .as_ref()
1371                        .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1372                    "operator-pinned recovery run"
1373                );
1374                ensure!(
1375                    !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1376                    "selected replacement recovery source"
1377                );
1378                let _owner = crate::pr_head::owner_lock(home, id)?;
1379                let mut state = RunState::load_under(id, home)?;
1380                ensure!(
1381                    state.repo == authority.repo,
1382                    "repository differs from recovery authority"
1383                );
1384                ensure!(
1385                    !crate::pr_recovery::explicit_sources_retained(home, id),
1386                    "explicitly retained recovery source"
1387                );
1388                ensure!(
1389                    state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1390                        == Liveness::Dead,
1391                    "run live or ownership unknown"
1392                );
1393                let _driver = crate::driver::maintenance(home, id)?;
1394                ensure!(
1395                    crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1396                    "within grace period"
1397                );
1398                ensure!(
1399                    !open.iter().any(|q| q.run == *id && q.status.open()),
1400                    "unanswered owner decision"
1401                );
1402                ensure!(
1403                    state.pr.is_none()
1404                        && state.pr_head.is_none()
1405                        && state.pr_recovery.is_none()
1406                        && state.land_approval.is_none()
1407                        && state.merge_decision.is_none()
1408                        && state.reviews.is_empty()
1409                        && state.recovery.is_none()
1410                        && state.candidate_preservation.is_empty()
1411                        && state.interrupted_implementations.is_empty()
1412                        && !state.released(),
1413                    "PR ownership or recovery checkpoint"
1414                );
1415                let proof = prove_empty(home, &state, latest).await?;
1416                commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1417                // Durable retirement precedes removal. A crash retries removal
1418                // using the same proof; it cannot make this attempt resumable.
1419                for path in &proof.paths {
1420                    ensure!(
1421                        git::worktree_remove_clean(&state.repo, path).await?,
1422                        "Git refused clean worktree removal"
1423                    );
1424                }
1425                Ok::<_, anyhow::Error>(())
1426            }
1427            .await;
1428            match result {
1429                Ok(()) => report.retired += 1,
1430                Err(e) => report.retained.push(format!("{id}: {e:#}")),
1431            }
1432        }
1433    }
1434}
1435
1436struct EmptyProof {
1437    paths: Vec<PathBuf>,
1438    _outputs: Vec<OutputLease>,
1439    _cargo: Vec<File>,
1440}
1441
1442// Compare actual source bytes with the original tree independently of index
1443// stat caches, assume-unchanged flags, sparse checkout and fsmonitor hints.
1444pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1445    let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1446    let mut files = Vec::new();
1447    let mut blobs = Vec::new();
1448    for record in tree.split('\0').filter(|s| !s.is_empty()) {
1449        let (header, name) = record.split_once('\t').context("unknown tree record")?;
1450        let parts: Vec<_> = header.split_whitespace().collect();
1451        ensure!(
1452            parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1453            "unsupported tracked source kind"
1454        );
1455        let path = worktree.join(name);
1456        let meta = std::fs::symlink_metadata(&path)?;
1457        ensure!(
1458            meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1459            "noncanonical tracked source"
1460        );
1461        #[cfg(unix)]
1462        {
1463            use std::os::unix::fs::PermissionsExt as _;
1464            ensure!(
1465                (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1466                "tracked executable mode differs from original base"
1467            );
1468        }
1469        path.to_str().context("non UTF-8 source")?;
1470        files.push(git::path_arg(&path));
1471        blobs.push(parts[2].to_owned());
1472    }
1473    for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1474        let mut args = vec!["hash-object", "--no-filters", "--"];
1475        args.extend(paths.iter().map(String::as_str));
1476        let actual = git::git(worktree, &args).await?;
1477        ensure!(
1478            actual.lines().eq(expected.iter().map(String::as_str)),
1479            "tracked bytes differ from original base"
1480        );
1481    }
1482    Ok(())
1483}
1484
1485async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1486    let id = &state.id;
1487    let paths = worktrees(state).await?;
1488    let mut guards = Vec::new();
1489    let mut cargo_guards = Vec::new();
1490    for path in &paths {
1491        guards.push(OutputLease::take(path, true)?);
1492        ensure!(
1493            git::rev_parse(path, "HEAD").await? == state.base_commit,
1494            "unique or unrecorded commits"
1495        );
1496        ensure!(
1497            git::git(
1498                path,
1499                &[
1500                    "--no-optional-locks",
1501                    "status",
1502                    "--porcelain",
1503                    "--untracked-files=all"
1504                ]
1505            )
1506            .await?
1507            .is_empty(),
1508            "tracked or untracked source changes"
1509        );
1510        ensure!(
1511            git::git(
1512                path,
1513                &[
1514                    "--no-optional-locks",
1515                    "status",
1516                    "--porcelain",
1517                    "--untracked-files=all",
1518                    "--ignored",
1519                    "--",
1520                    ".",
1521                    ":(exclude)target"
1522                ]
1523            )
1524            .await?
1525            .is_empty(),
1526            "source changes, ignored files or build outputs"
1527        );
1528        tracked_base(path, &state.base_commit).await?;
1529        match std::fs::symlink_metadata(path.join("target")) {
1530            Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1531            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1532            Err(error) => return Err(error.into()),
1533        }
1534        for checkpoint in [
1535            "rebase-merge",
1536            "rebase-apply",
1537            "sequencer",
1538            "BISECT_LOG",
1539            "REVERT_HEAD",
1540            "AUTO_MERGE",
1541            "MERGE_MSG",
1542            "CHERRY_PICK_HEAD",
1543            "MERGE_HEAD",
1544            "REBASE_HEAD",
1545        ] {
1546            let location = git::git(
1547                path,
1548                &[
1549                    "rev-parse",
1550                    "--path-format=absolute",
1551                    "--git-path",
1552                    checkpoint,
1553                ],
1554            )
1555            .await?;
1556            ensure!(
1557                matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1558                "retained Git recovery metadata: {checkpoint}"
1559            );
1560        }
1561        // Git also initializes ORIG_HEAD when creating a worktree.
1562        // Only an origin outside the base is a recovery source.
1563        let origin_path = git::git(
1564            path,
1565            &[
1566                "rev-parse",
1567                "--path-format=absolute",
1568                "--git-path",
1569                "ORIG_HEAD",
1570            ],
1571        )
1572        .await?;
1573        match std::fs::symlink_metadata(origin_path.trim()) {
1574            Ok(meta) => {
1575                ensure!(
1576                    meta.is_file() && !meta.file_type().is_symlink(),
1577                    "unknown ORIG_HEAD artifact"
1578                );
1579                git::rev_parse(path, "ORIG_HEAD")
1580                    .await
1581                    .context("unreadable ORIG_HEAD")?;
1582            }
1583            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1584            Err(e) => return Err(e.into()),
1585        }
1586        if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1587            ensure!(
1588                git::git_raw(
1589                    path,
1590                    &["merge-base", "--is-ancestor", &original, &state.base_commit]
1591                )
1592                .await?
1593                .ok(),
1594                "ORIG_HEAD retains commits outside the original base"
1595            );
1596        }
1597        let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1598        for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1599            ensure!(
1600                git::git_raw(
1601                    path,
1602                    &["merge-base", "--is-ancestor", head, &state.base_commit]
1603                )
1604                .await?
1605                .ok(),
1606                "worktree reflog retains commits outside the original base"
1607            );
1608        }
1609        for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1610            ensure!(
1611                !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1612                    .await?
1613                    .ok(),
1614                "unfinished Git operation"
1615            );
1616        }
1617    }
1618    // Missing worktrees cannot establish that unrecorded prep edits
1619    // were empty. Recorded candidates must also point at the base.
1620    if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1621        return Ok(EmptyProof {
1622            paths,
1623            _outputs: guards,
1624            _cargo: cargo_guards,
1625        });
1626    }
1627    ensure!(!paths.is_empty(), "no worktrees to prove empty");
1628    for candidate in &state.candidates {
1629        ensure!(
1630            (paths.contains(&candidate.worktree)
1631                || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1632                && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1633            "candidate recovery source"
1634        );
1635    }
1636    let pattern = format!("refs/heads/magi/{}/", state.short());
1637    let branches = git::git(
1638        &state.repo,
1639        &["for-each-ref", "--format=%(objectname)", &pattern],
1640    )
1641    .await?;
1642    ensure!(
1643        branches.lines().all(|head| head == state.base_commit),
1644        "unrecorded candidate branch changes"
1645    );
1646    Ok(EmptyProof {
1647        paths,
1648        _outputs: guards,
1649        _cargo: cargo_guards,
1650    })
1651}
1652
1653/// The only task-write critical section in retirement. All Git proof and
1654/// deletion happens outside it, under the run's owner/driver/output leases.
1655/// Atomic task writes make the earlier snapshot consistent; this final compare
1656/// rejects every intervening owner selection, hold, edit or history update.
1657fn commit_retirement(
1658    home: &Path,
1659    queue: &crate::queue::Queue,
1660    expected: &crate::queue::Task,
1661    state: &mut RunState,
1662    authority: &str,
1663    sources: &[String],
1664    now: Timestamp,
1665) -> Result<()> {
1666    let _write = queue.lock_task(&expected.id)?;
1667    let current = queue.get(&expected.id)?;
1668    ensure!(
1669        serde_json::to_value(&current)? == serde_json::to_value(expected)?,
1670        "task changed during retirement proof; retaining worktrees for the current owner decision"
1671    );
1672    ensure!(
1673        !current
1674            .resume_override
1675            .as_ref()
1676            .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1677            && !current
1678                .replacement
1679                .as_ref()
1680                .is_some_and(|pin| pin.run == state.id),
1681        "current owner selected this recovery source"
1682    );
1683    ensure!(
1684        !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1685        "explicitly retained recovery source"
1686    );
1687    ensure!(
1688        !crate::ask::Questions::at(home.join("questions"))
1689            .list()
1690            .iter()
1691            .any(|q| q.status.open()
1692                && (q.run == state.id
1693                    || q.run == current.id
1694                    || current.blocked_by.contains(&q.id))),
1695        "owner decision arrived during retirement proof"
1696    );
1697    let dir = home.join("recovery-lineages");
1698    std::fs::create_dir_all(&dir)?;
1699    let tmp = dir.join(format!("{}.tmp", current.id));
1700    let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1701        "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1702    std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1703    std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1704    state.retired_to = Some(authority.to_owned());
1705    state.event(
1706        "retention",
1707        format!(
1708            "empty attempt retired to {authority}; recovery sources: {}",
1709            sources.join(", ")
1710        ),
1711    );
1712    state.save_under(home)?;
1713    // No Git command or worktree removal may be added before this guard drops.
1714    Ok(())
1715}
1716
1717/// Wait for the emergency reserve to be exhausted. The normal floor requests
1718/// a checkpoint first; half the floor stops owned writers before node completion.
1719/// A failed measurement also closes admission rather than allowing blind writes.
1720pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1721    if min_free == 0 {
1722        return std::future::pending().await;
1723    }
1724    loop {
1725        for path in &paths {
1726            let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1727            match crate::disk::free_bytes(existing) {
1728                Ok(free) if free >= min_free / 2 => {}
1729                Ok(free) => {
1730                    return format!(
1731                        "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1732                    );
1733                }
1734                Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1735            }
1736        }
1737        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1738    }
1739}
1740
1741#[cfg(all(test, unix))]
1742mod tests {
1743    use super::*;
1744    use crate::{
1745        config::{Config, Disk},
1746        queue::{Source, Task},
1747        run::{Candidate, RunStatus},
1748    };
1749
1750    struct Fx {
1751        _temp: tempfile::TempDir,
1752        home: PathBuf,
1753        repo: PathBuf,
1754        bay: PathBuf,
1755        base: String,
1756    }
1757    impl Fx {
1758        async fn new() -> Self {
1759            let temp = tempfile::tempdir().unwrap();
1760            let root = temp.path().canonicalize().unwrap();
1761            let home = root.join("home");
1762            let repo = root.join("repo");
1763            let bay = root.join("worktrees");
1764            std::fs::create_dir_all(&repo).unwrap();
1765            std::fs::create_dir_all(home.join("runs")).unwrap();
1766            git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1767            git::git(&repo, &["config", "user.name", "Fixture"])
1768                .await
1769                .unwrap();
1770            git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1771                .await
1772                .unwrap();
1773            git::git(&repo, &["config", "commit.gpgsign", "false"])
1774                .await
1775                .unwrap();
1776            std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1777            std::fs::write(repo.join("source"), "base\n").unwrap();
1778            git::git(&repo, &["add", "."]).await.unwrap();
1779            git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1780            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1781            Self {
1782                _temp: temp,
1783                home,
1784                repo,
1785                bay,
1786                base,
1787            }
1788        }
1789        async fn run(&self, tag: &str, useful: bool) -> RunState {
1790            let mut cfg = Config::default();
1791            cfg.graph.worktree_root = Some(self.bay.clone());
1792            let mut state = RunState::new(
1793                self.repo.clone(),
1794                "main".into(),
1795                self.base.clone(),
1796                "task".into(),
1797                cfg,
1798            );
1799            state.id = format!("20260101-000000-{tag}");
1800            state.status = RunStatus::Blocked;
1801            state.driver_exited = true;
1802            let wt = state.worktree_root().join("cand-A");
1803            let branch = format!("magi/{tag}/A");
1804            git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1805                .await
1806                .unwrap();
1807            if useful {
1808                std::fs::write(wt.join("source"), tag).unwrap();
1809                git::git(&wt, &["add", "."]).await.unwrap();
1810                git::git(&wt, &["commit", "-m", "candidate change"])
1811                    .await
1812                    .unwrap();
1813            }
1814            state.candidates.push(Candidate {
1815                index: 0,
1816                label: 'A',
1817                agent: "fixture".into(),
1818                branch,
1819                worktree: wt,
1820                summary: String::new(),
1821                stat: String::new(),
1822                files: usize::from(useful),
1823                commits: usize::from(useful),
1824                empty: !useful,
1825                failed: None,
1826                verified_noop: None,
1827                duration_ms: 0,
1828                folded: false,
1829            });
1830            state.save_under(&self.home).unwrap();
1831            state
1832        }
1833        fn disk() -> Disk {
1834            Disk {
1835                worktree_limit_bytes: 1,
1836                worktree_reclaim_bytes: 1024 * 1024,
1837                fold_grace_secs: 0,
1838                min_free_bytes: 0,
1839                ..Disk::default()
1840            }
1841        }
1842        fn target(wt: &Path) {
1843            let target = wt.join("target");
1844            std::fs::create_dir_all(target.join("debug")).unwrap();
1845            let mut artifact = [1; 200];
1846            artifact[..4].copy_from_slice(b"\x7fELF");
1847            std::fs::write(target.join("debug/output"), artifact).unwrap();
1848            let output = target
1849                .join("debug/output")
1850                .display()
1851                .to_string()
1852                .replace('\\', "\\\\")
1853                .replace(' ', "\\ ");
1854            std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1855            std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1856            std::fs::write(
1857                target.join("CACHEDIR.TAG"),
1858                "Signature: 8a477f597d28d172789f06886806bc55\n",
1859            )
1860            .unwrap();
1861            std::fs::write(
1862                target.join(".rustc_info.json"),
1863                "{\"rustc_fingerprint\":123}",
1864            )
1865            .unwrap();
1866        }
1867        fn now() -> Timestamp {
1868            Timestamp::now() + jiff::SignedDuration::from_secs(2)
1869        }
1870        fn task(&self, runs: &[RunState]) -> Task {
1871            let mut task = Task::new(
1872                "fixture".into(),
1873                "task".into(),
1874                self.repo.clone(),
1875                Source::Human,
1876            );
1877            task.runs = runs.iter().map(|s| s.id.clone()).collect();
1878            crate::queue::Queue::at(self.home.join("queue"))
1879                .put(&mut task)
1880                .unwrap();
1881            task
1882        }
1883    }
1884
1885    #[cfg(unix)]
1886    #[tokio::test]
1887    async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1888        let f = Fx::new().await;
1889        let state = f.run("ca01", true).await;
1890        let wt = &state.candidates[0].worktree;
1891        let generation = crate::incremental::tests::generate(wt);
1892        // Remove dep-info authority: incremental proof must be independent.
1893        for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1894            for entry in std::fs::read_dir(dir).unwrap() {
1895                let path = entry.unwrap().path();
1896                if path.extension().is_some_and(|x| x == "d") {
1897                    std::fs::remove_file(path).unwrap();
1898                }
1899            }
1900        }
1901        assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1902        assert_eq!(
1903            sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1904            0,
1905            "live generation is protected"
1906        );
1907        drop(generation);
1908        let candidates = crate::incremental::candidates(wt).unwrap();
1909        let logical: u64 = candidates
1910            .files
1911            .iter()
1912            .map(|p| std::fs::metadata(p).unwrap().len())
1913            .sum();
1914        let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1915        std::fs::write(&unknown, "retain unknown data").unwrap();
1916        drop(candidates);
1917        let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1918        assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1919        assert!(
1920            refused
1921                .retained
1922                .iter()
1923                .any(|s| s.contains("provenance mismatch"))
1924        );
1925        std::fs::remove_file(&unknown).unwrap();
1926        let legacy = wt.join("target/debug/incremental/legacy-session");
1927        std::fs::create_dir(&legacy).unwrap();
1928        // Even compiler-authored bytes do not retroactively confer ownership.
1929        let candidates = crate::incremental::candidates(wt).unwrap();
1930        let bin = candidates
1931            .files
1932            .iter()
1933            .find(|p| p.extension().is_some_and(|x| x == "bin"))
1934            .unwrap();
1935        std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1936        drop(candidates);
1937        let cargo = cargo_locks(&wt.join("target")).unwrap();
1938        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1939        drop(cargo);
1940        let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1941        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1942        drop(outputs);
1943        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1944        assert_eq!(report.freed_bytes, logical, "{report:?}");
1945        assert_eq!(report.proven_reclaimable_bytes, logical);
1946        assert!(report.measured_bytes > logical);
1947        assert_eq!(
1948            report.protected_or_unproven_bytes,
1949            report.measured_bytes - logical
1950        );
1951        #[cfg(unix)]
1952        {
1953            assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
1954            assert!(
1955                report.measured_allocated_file_bytes.unwrap()
1956                    >= report.estimated_released_allocated_bytes.unwrap()
1957            );
1958        }
1959        assert!(report.observed_home_free_change_bytes.is_some());
1960        assert!(legacy.join("query-cache.bin").is_file());
1961        assert!(wt.join("src/lib.rs").is_file());
1962        assert_eq!(report.retired, 0);
1963    }
1964
1965    #[cfg(unix)]
1966    #[tokio::test]
1967    async fn incremental_and_deps_links_account_for_retained_third_link() {
1968        use std::os::unix::fs::MetadataExt;
1969        let f = Fx::new().await;
1970        let state = f.run("ca02", true).await;
1971        let wt = &state.candidates[0].worktree;
1972        let generation = crate::incremental::tests::generate(wt);
1973        drop(generation);
1974        let candidates = crate::incremental::candidates(wt).unwrap();
1975        let objects = candidates
1976            .files
1977            .iter()
1978            .filter(|p| p.extension().is_some_and(|x| x == "o"))
1979            .take(2)
1980            .cloned()
1981            .collect::<Vec<_>>();
1982        assert_eq!(
1983            objects.len(),
1984            2,
1985            "fixture must produce real incremental objects"
1986        );
1987        let mut outputs = candidates.files.clone();
1988        drop(candidates);
1989        let target = wt.join("target");
1990        for dir in [target.join("debug"), target.join("debug/deps")] {
1991            for entry in std::fs::read_dir(dir).unwrap() {
1992                let path = entry.unwrap().path();
1993                if path.extension().is_some_and(|x| x == "d") {
1994                    std::fs::remove_file(path).unwrap();
1995                }
1996            }
1997        }
1998        // Cargo also leaves hard-linked object outputs in deps. Remove those
1999        // fixture outputs so the only external links are the ones below.
2000        for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
2001            let path = entry.unwrap().path();
2002            if path.is_file() {
2003                std::fs::remove_file(path).unwrap();
2004            }
2005        }
2006        for (index, object) in objects.iter().enumerate() {
2007            assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2008            let dep = target.join(format!("debug/deps/link{index}"));
2009            std::fs::hard_link(object, &dep).unwrap();
2010            std::fs::write(
2011                dep.with_extension("d"),
2012                format!("{}: src/lib.rs\n", dep.display()),
2013            )
2014            .unwrap();
2015            outputs.push(dep);
2016        }
2017        let retained = wt.join("retained-input");
2018        std::fs::hard_link(&objects[1], &retained).unwrap();
2019        let sentinel = std::fs::read(&retained).unwrap();
2020        let logical: u64 = outputs
2021            .iter()
2022            .map(|p| std::fs::metadata(p).unwrap().len())
2023            .sum();
2024        let mut seen = std::collections::BTreeSet::new();
2025        let retained_meta = std::fs::metadata(&retained).unwrap();
2026        let retained_inode = (retained_meta.dev(), retained_meta.ino());
2027        let mut expected = 0u64;
2028        for output in &outputs {
2029            let m = std::fs::metadata(output).unwrap();
2030            let inode = (m.dev(), m.ino());
2031            if inode != retained_inode && seen.insert(inode) {
2032                expected += m.blocks() * 512;
2033            }
2034        }
2035        assert!(expected > 0);
2036        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2037        assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2038        assert_eq!(report.freed_bytes, logical, "{report:?}");
2039        assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2040        assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2041        assert!(outputs.iter().all(|p| !p.exists()));
2042        assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2043        assert!(wt.join("src/lib.rs").is_file());
2044    }
2045
2046    #[tokio::test]
2047    async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2048        let f = Fx::new().await;
2049        let state = f.run("ca03", true).await;
2050        let wt = &state.candidates[0].worktree;
2051        Fx::target(wt);
2052        std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2053        let unknown = wt.join("target/debug/operator-note");
2054        std::fs::write(&unknown, vec![1; 4096]).unwrap();
2055        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2056        assert_eq!(report.proven_reclaimable_bytes, 0);
2057        #[cfg(unix)]
2058        assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2059        assert!(report.protected_or_unproven_bytes >= 4096);
2060        let message = || {
2061            crate::notices::Notices::at(f.home.join("notifications"))
2062                .list()
2063                .into_iter()
2064                .find(|n| n.key == "disk:retention")
2065                .unwrap()
2066                .message
2067        };
2068        let notice = message();
2069        assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2070        assert!(notice.contains("protected or unproven"), "{notice}");
2071        assert!(
2072            notice.contains("legacy incremental/build and unknown files retained"),
2073            "{notice}"
2074        );
2075        let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2076        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2077        assert_eq!(report.freed_bytes, 0);
2078        let notice = message();
2079        assert!(notice.contains(&state.id), "{notice}");
2080        assert!(
2081            notice.contains("lock") || notice.contains("owner"),
2082            "{notice}"
2083        );
2084        drop(owner);
2085        assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2086    }
2087
2088    #[tokio::test]
2089    async fn real_cargo_build_holds_the_target_against_reclamation() {
2090        let f = Fx::new().await;
2091        let state = f.run("ca02", true).await;
2092        let wt = &state.candidates[0].worktree;
2093        Fx::target(wt);
2094        std::fs::create_dir(wt.join("src")).unwrap();
2095        std::fs::write(
2096            wt.join("Cargo.toml"),
2097            "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2098        )
2099        .unwrap();
2100        std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2101        std::fs::write(
2102            wt.join("build.rs"),
2103            r#"fn main() {
2104            let ready = std::env::var("FIXTURE_READY").unwrap();
2105            let release = std::env::var("FIXTURE_RELEASE").unwrap();
2106            std::fs::write(ready, "ready").unwrap();
2107            let start = std::time::Instant::now();
2108            while !std::path::Path::new(&release).exists() && start.elapsed().as_secs() < 15 {
2109                std::thread::sleep(std::time::Duration::from_millis(10));
2110            }
2111        }"#,
2112        )
2113        .unwrap();
2114        let ready = wt.join("ready");
2115        let release = wt.join("release");
2116        let mut cargo = tokio::process::Command::new("cargo");
2117        cargo.env_clear();
2118        for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2119            if let Some(value) = std::env::var_os(key) {
2120                cargo.env(key, value);
2121            }
2122        }
2123        let mut child = cargo
2124            .current_dir(wt)
2125            .env("RUSTUP_TOOLCHAIN", "stable")
2126            .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2127            .env("CARGO_INCREMENTAL", "1")
2128            .env("FIXTURE_READY", &ready)
2129            .env("FIXTURE_RELEASE", &release)
2130            .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2131            .arg(wt.join("target"))
2132            .stdout(std::process::Stdio::null())
2133            .stderr(std::process::Stdio::null())
2134            .kill_on_drop(true)
2135            .spawn()
2136            .unwrap();
2137        let arrived = tokio::time::timeout(std::time::Duration::from_secs(10), async {
2138            while !ready.exists() {
2139                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2140            }
2141        })
2142        .await
2143        .is_ok();
2144        let report = if arrived {
2145            Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2146        } else {
2147            None
2148        };
2149        // Release and join our build before making assertions, including on failure.
2150        std::fs::write(release, "release").unwrap();
2151        let status = tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await;
2152        if status.is_err() {
2153            child.kill().await.unwrap();
2154            child.wait().await.unwrap();
2155        }
2156        assert!(
2157            arrived,
2158            "fixture build never reached its active build script"
2159        );
2160        assert!(status.unwrap().unwrap().success());
2161        let report = report.unwrap();
2162        assert_eq!(report.freed_bytes, 0, "{report:?}");
2163        assert!(
2164            report
2165                .retained
2166                .iter()
2167                .any(|s| s.contains("Cargo build is active")),
2168            "{report:?}"
2169        );
2170        assert!(wt.join("target/debug/output").is_file());
2171        assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2172    }
2173
2174    #[cfg(unix)]
2175    #[test]
2176    fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2177        let temp = tempfile::tempdir().unwrap();
2178        let a = temp.path().join("a");
2179        let b = temp.path().join("b");
2180        let outside = temp.path().join("retained-input");
2181        std::fs::write(&a, vec![1; 65536]).unwrap();
2182        std::fs::hard_link(&a, &b).unwrap();
2183        std::fs::hard_link(&a, &outside).unwrap();
2184        let mut seen = Default::default();
2185        let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2186        assert!(blocks > 0);
2187        let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2188        std::fs::remove_file(&a).unwrap();
2189        assert_eq!(accounting.unlinked(&a), Some(0));
2190        std::fs::remove_file(&b).unwrap();
2191        assert_eq!(accounting.unlinked(&b), Some(0));
2192        assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2193        let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2194        std::fs::remove_file(&outside).unwrap();
2195        assert_eq!(accounting.unlinked(&outside), Some(blocks));
2196    }
2197
2198    #[cfg(unix)]
2199    #[test]
2200    fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2201        let temp = tempfile::tempdir_in(".").unwrap();
2202        let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2203        assert!(home.is_relative());
2204        let paths = vec![std::env::current_dir().unwrap()];
2205        let gib = 1024 * 1024 * 1024;
2206        let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2207        assert!(
2208            OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2209                9 * gib
2210            ))
2211            .is_err()
2212        );
2213        drop(reservation);
2214        assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2215        let bad = temp.path().join("symlink-state");
2216        std::fs::create_dir(&bad).unwrap();
2217        std::os::unix::fs::symlink(
2218            home.canonicalize().unwrap().join("disk-admission"),
2219            bad.join("disk-admission"),
2220        )
2221        .unwrap();
2222        assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2223    }
2224
2225    #[cfg(unix)]
2226    #[test]
2227    fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2228        let temp = tempfile::tempdir().unwrap();
2229        let home = temp.path().canonicalize().unwrap();
2230        let paths = vec![home.clone(), home.join("not-created-yet")];
2231        let gib = 1024 * 1024 * 1024;
2232        let floor = 8 * gib;
2233        let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2234        assert_eq!(first.len(), 1);
2235        let blocked =
2236            OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2237        assert!(blocked.to_string().contains("active writer reservations"));
2238        let second =
2239            OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2240        drop(first);
2241        assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2242        drop(second);
2243        assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2244        assert!(
2245            OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2246        );
2247    }
2248
2249    #[cfg(unix)]
2250    #[test]
2251    fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2252        let temp = tempfile::tempdir().unwrap();
2253        let home = temp.path().canonicalize().unwrap();
2254        let barrier = std::sync::Barrier::new(2);
2255        let gib = 1024 * 1024 * 1024;
2256        let results = std::thread::scope(|scope| {
2257            let threads: Vec<_> = (0..2)
2258                .map(|_| {
2259                    scope.spawn(|| {
2260                        barrier.wait();
2261                        OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2262                            Ok(100 + gib)
2263                        })
2264                    })
2265                })
2266                .collect();
2267            threads
2268                .into_iter()
2269                .map(|thread| thread.join().unwrap())
2270                .collect::<Vec<_>>()
2271        });
2272        assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2273        assert!(
2274            results
2275                .iter()
2276                .filter_map(|r| r.as_ref().err())
2277                .all(|e| e.to_string().contains("active writer reservations"))
2278        );
2279    }
2280
2281    #[tokio::test]
2282    async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2283        real_cargo_fixture(false).await;
2284    }
2285
2286    #[tokio::test]
2287    async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2288        real_cargo_fixture(true).await;
2289    }
2290
2291    async fn real_cargo_fixture(precreate: bool) {
2292        let f = Fx::new().await;
2293        let state = f.run("ca00", true).await;
2294        let wt = &state.candidates[0].worktree;
2295        let source = "fn main() { println!(\"retained fixture\"); }\n";
2296        std::fs::create_dir(wt.join("src")).unwrap();
2297        std::fs::write(wt.join("src/main.rs"), source).unwrap();
2298        std::fs::write(
2299            wt.join("Cargo.toml"),
2300            "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2301        )
2302        .unwrap();
2303        if precreate {
2304            std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2305        }
2306        let mut cargo = std::process::Command::new("cargo");
2307        cargo.env_clear();
2308        for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2309            if let Some(value) = std::env::var_os(key) {
2310                cargo.env(key, value);
2311            }
2312        }
2313        let build = cargo
2314            .current_dir(wt)
2315            .env("RUSTUP_TOOLCHAIN", "stable")
2316            .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2317            .env("CARGO_INCREMENTAL", "0")
2318            .args([
2319                "build",
2320                "--offline",
2321                "--release",
2322                "--jobs",
2323                "1",
2324                "--target-dir",
2325            ])
2326            .arg(wt.join("target"))
2327            .output()
2328            .unwrap();
2329        assert!(
2330            build.status.success(),
2331            "{}",
2332            String::from_utf8_lossy(&build.stderr)
2333        );
2334        assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2335        if precreate {
2336            // Cargo leaves pre-existing target roots untagged on subsequent builds too.
2337            let again = cargo.output().unwrap();
2338            assert!(
2339                again.status.success(),
2340                "{}",
2341                String::from_utf8_lossy(&again.stderr)
2342            );
2343            assert!(!wt.join("target/CACHEDIR.TAG").exists());
2344            assert!(cargo_target(wt).await.is_err());
2345        }
2346        git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2347            .await
2348            .unwrap();
2349        git::git(wt, &["commit", "-m", "real Cargo fixture"])
2350            .await
2351            .unwrap();
2352        let artifact = wt.join("target/release/retained-fixture");
2353        assert!(artifact.is_file());
2354        let unknown = wt.join("target/release/operator-note");
2355        std::fs::write(&unknown, "keep this note\n").unwrap();
2356
2357        let target = wt.join("target");
2358        let mut preserved = std::collections::BTreeMap::new();
2359        for relative in [
2360            "tmp/sentinel",
2361            "magi-tools/sentinel",
2362            "lifecycle-client-check.log",
2363        ] {
2364            let path = target.join(relative);
2365            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2366            std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2367            preserved.insert(path.clone(), std::fs::read(path).unwrap());
2368        }
2369        // Even genuine compiler bytes and valid dep-info outside supported
2370        // profiles cannot expand the deletion inventory.
2371        let decoy = target.join("magi-tools/retained-fixture");
2372        std::fs::copy(&artifact, &decoy).unwrap();
2373        let dep_info = decoy.with_extension("d");
2374        std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2375        for path in [
2376            decoy,
2377            dep_info,
2378            wt.join("Cargo.toml"),
2379            wt.join("Cargo.lock"),
2380        ] {
2381            preserved.insert(path.clone(), std::fs::read(path).unwrap());
2382        }
2383        let proven = declared_outputs(&target).unwrap();
2384
2385        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2386        assert!(out.freed_bytes > 0, "{out:?}");
2387        for (path, bytes) in preserved {
2388            assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2389        }
2390        assert!(!artifact.exists(), "{out:?}");
2391        assert_eq!(
2392            std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2393            source
2394        );
2395        assert_eq!(
2396            std::fs::read_to_string(&unknown).unwrap(),
2397            "keep this note\n"
2398        );
2399        assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2400        let ledger: serde_json::Value = serde_json::from_slice(
2401            &std::fs::read(
2402                f.home
2403                    .join("runs")
2404                    .join(&state.id)
2405                    .join("build-outputs.json"),
2406            )
2407            .unwrap(),
2408        )
2409        .unwrap();
2410        assert_eq!(
2411            ledger["identity"],
2412            if precreate {
2413                "rustc-profile-dep-info"
2414            } else {
2415                "cache-tag"
2416            }
2417        );
2418        if precreate {
2419            assert!(!target.join("CACHEDIR.TAG").exists());
2420            assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2421        }
2422        let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2423        assert!(deleted.contains(&artifact));
2424        assert_eq!(deleted, proven);
2425        for path in deleted {
2426            assert!(!path.exists());
2427            assert!(
2428                [target.join("release"), target.join("release/deps")]
2429                    .contains(&path.parent().unwrap().to_owned())
2430            );
2431        }
2432        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2433    }
2434
2435    fn missing_tag_target(wt: &Path) {
2436        Fx::target(wt);
2437        let target = wt.join("target");
2438        std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2439        for subdir in ["debug/deps", "debug/.fingerprint"] {
2440            std::fs::create_dir_all(target.join(subdir)).unwrap();
2441        }
2442        std::fs::write(target.join(".rustc_info.json"),
2443            r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2444    }
2445
2446    #[tokio::test]
2447    async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2448        for legacy in [false, true] {
2449            let f = Fx::new().await;
2450            let mut state = f.run("ac00", true).await;
2451            Fx::target(&state.candidates[0].worktree);
2452            let original = state.updated_at;
2453            let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2454            if legacy {
2455                let path = f.home.join("runs").join(&state.id).join("run.json");
2456                let mut value = serde_json::to_value(&state).unwrap();
2457                value.as_object_mut().unwrap().remove("output_activity_at");
2458                std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2459                state = RunState::load_under(&state.id, &f.home).unwrap();
2460                assert_eq!(state.output_activity(), original);
2461            }
2462            let age = state.output_activity();
2463            for offset in [30, 20, 10] {
2464                state
2465                    .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2466                    .unwrap();
2467                state = RunState::load_under(&state.id, &f.home).unwrap();
2468                assert_eq!(state.output_activity(), age);
2469            }
2470            let mut disk = Fx::disk();
2471            disk.fold_grace_secs = 3600;
2472            let report = sweep(&f.home, &disk, now).await;
2473            assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2474        }
2475    }
2476
2477    #[tokio::test]
2478    async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2479        for evidence in [
2480            "execution",
2481            "artifact",
2482            "dep-info",
2483            "fingerprint",
2484            "lease",
2485            "future",
2486        ] {
2487            let f = Fx::new().await;
2488            let mut state = f.run("ac01", true).await;
2489            let wt = state.candidates[0].worktree.clone();
2490            missing_tag_target(&wt);
2491            let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2492            let recent = now - jiff::SignedDuration::from_secs(10);
2493            match evidence {
2494                "execution" => {
2495                    state.note_output_activity(recent);
2496                    state.save_under_at(&f.home, recent).unwrap();
2497                }
2498                "lease" => {
2499                    let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2500                    let file = lease._file.try_clone().unwrap();
2501                    drop(lease);
2502                    file.set_modified(std::time::SystemTime::from(recent))
2503                        .unwrap();
2504                }
2505                _ => {
2506                    let relative = match evidence {
2507                        "dep-info" => "debug/output.d",
2508                        "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2509                        _ => "debug/output",
2510                    };
2511                    let path = wt.join("target").join(relative);
2512                    let file = OpenOptions::new()
2513                        .write(true)
2514                        .create(true)
2515                        .truncate(false)
2516                        .open(path)
2517                        .unwrap();
2518                    let at = if evidence == "future" {
2519                        now + jiff::SignedDuration::from_secs(10)
2520                    } else {
2521                        recent
2522                    };
2523                    file.set_modified(std::time::SystemTime::from(at)).unwrap();
2524                }
2525            }
2526            let mut disk = Fx::disk();
2527            disk.fold_grace_secs = 3600;
2528            let report = sweep(&f.home, &disk, now).await;
2529            assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2530            assert!(
2531                report.retained.iter().any(|s| s.contains("grace period")
2532                    || s.contains("execution activity after inventory")),
2533                "{evidence}: {report:?}"
2534            );
2535            let aged = now + jiff::SignedDuration::from_secs(7200);
2536            let report = sweep(&f.home, &disk, aged).await;
2537            assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2538        }
2539    }
2540
2541    #[tokio::test]
2542    async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2543        let f = Fx::new().await;
2544        let state = f.run("ac02", true).await;
2545        let wt = &state.candidates[0].worktree;
2546        missing_tag_target(wt);
2547        let target = wt.join("target");
2548        let unknown = target.join("debug/notes");
2549        std::fs::write(&unknown, "operator data").unwrap();
2550        for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2551            let path = target.join("debug").join(lock);
2552            let file = OpenOptions::new()
2553                .create(true)
2554                .truncate(false)
2555                .read(true)
2556                .write(true)
2557                .open(&path)
2558                .unwrap();
2559            crate::pr_head::try_lock_file(&file).unwrap();
2560            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2561            assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2562            assert!(
2563                report
2564                    .retained
2565                    .iter()
2566                    .any(|s| s.contains(path.to_str().unwrap())),
2567                "{report:?}"
2568            );
2569        }
2570        let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2571        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2572        drop(owner);
2573        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2574        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2575        drop(driver);
2576        let output = OutputLease::borrow(wt).unwrap().unwrap();
2577        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2578        drop(output);
2579        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2580        assert_eq!(report.freed_bytes, 200, "{report:?}");
2581        assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2582        assert!(!target.join("CACHEDIR.TAG").exists());
2583        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2584        assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2585        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2586    }
2587
2588    #[tokio::test]
2589    async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2590        let f = Fx::new().await;
2591        let state = f.run("ac04", true).await;
2592        let wt = &state.candidates[0].worktree;
2593        missing_tag_target(wt);
2594        let target = wt.join("target");
2595        let first = target.join("debug/output");
2596        let second = target.join("debug/second");
2597        std::fs::copy(&first, &second).unwrap();
2598        std::fs::write(
2599            second.with_extension("d"),
2600            format!("{}: source\n", second.display()),
2601        )
2602        .unwrap();
2603        let outputs = declared_outputs(&target).unwrap();
2604        assert_eq!(outputs.len(), 2);
2605        // Simulate interruption after durable adoption and one successful unlink.
2606        std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2607            serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2608        std::fs::remove_file(first).unwrap();
2609        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2610        assert_eq!(report.freed_bytes, 200, "{report:?}");
2611        assert!(!second.exists());
2612        assert!(second.with_extension("d").exists());
2613        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2614    }
2615
2616    #[tokio::test]
2617    async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2618        for invalid in [
2619            "identity",
2620            "deps",
2621            "fingerprint",
2622            "lock",
2623            "dep-info",
2624            "tag-symlink",
2625            "bad-tag",
2626        ] {
2627            let f = Fx::new().await;
2628            let state = f.run("ac03", true).await;
2629            let wt = &state.candidates[0].worktree;
2630            missing_tag_target(wt);
2631            let target = wt.join("target");
2632            match invalid {
2633                "identity" => std::fs::write(
2634                    target.join(".rustc_info.json"),
2635                    r#"{"rustc_fingerprint":123}"#,
2636                )
2637                .unwrap(),
2638                "deps" | "fingerprint" => {
2639                    let path = target.join("debug").join(if invalid == "deps" {
2640                        "deps"
2641                    } else {
2642                        ".fingerprint"
2643                    });
2644                    std::fs::remove_dir(&path).unwrap();
2645                    std::os::unix::fs::symlink(&f.repo, path).unwrap();
2646                }
2647                "lock" => {
2648                    std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2649                }
2650                "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2651                "tag-symlink" => {
2652                    std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2653                        .unwrap()
2654                }
2655                "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2656                _ => unreachable!(),
2657            }
2658            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2659            assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2660            assert!(target.join("debug/output").is_file());
2661        }
2662    }
2663
2664    #[tokio::test]
2665    async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2666        let f = Fx::new().await;
2667        let first = f.run("ba00", true).await;
2668        let second = f.run("ba01", true).await;
2669        for state in [&first, &second] {
2670            Fx::target(&state.candidates[0].worktree);
2671        }
2672        let mut disk = Fx::disk();
2673        disk.worktree_reclaim_bytes = 100;
2674        let out = sweep(&f.home, &disk, Fx::now()).await;
2675        assert_eq!(out.freed_bytes, 200, "{out:?}");
2676        assert!(
2677            out.retained
2678                .contains(&format!("{}: per-pass byte budget spent", second.id)),
2679            "{out:?}"
2680        );
2681        assert!(
2682            !first.candidates[0]
2683                .worktree
2684                .join("target/debug/output")
2685                .exists()
2686        );
2687        assert!(
2688            second.candidates[0]
2689                .worktree
2690                .join("target/debug/output")
2691                .exists()
2692        );
2693        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2694    }
2695
2696    #[tokio::test]
2697    async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2698        let f = Fx::new().await;
2699        let small = f.run("bb00", true).await;
2700        let large = f.run("bb01", true).await;
2701        for state in [&small, &large] {
2702            Fx::target(&state.candidates[0].worktree);
2703        }
2704        let output = large.candidates[0].worktree.join("target/debug/output");
2705        std::fs::OpenOptions::new()
2706            .write(true)
2707            .open(&output)
2708            .unwrap()
2709            .set_len(1000)
2710            .unwrap();
2711        let mut disk = Fx::disk();
2712        disk.worktree_reclaim_bytes = 300;
2713        disk.min_free_bytes = u64::MAX;
2714        // Pressure must trigger reclamation even when aggregate usage is below cap.
2715        disk.worktree_limit_bytes = u64::MAX;
2716        let out = sweep(&f.home, &disk, Fx::now()).await;
2717        assert_eq!(out.freed_bytes, 1000, "{out:?}");
2718        assert!(!output.exists());
2719        assert!(
2720            small.candidates[0]
2721                .worktree
2722                .join("target/debug/output")
2723                .exists()
2724        );
2725    }
2726
2727    #[tokio::test]
2728    async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2729        let f = Fx::new().await;
2730        let first = f.run("bc00", true).await;
2731        let second = f.run("bc01", true).await;
2732        for state in [&first, &second] {
2733            Fx::target(&state.candidates[0].worktree);
2734        }
2735        // Non-declared Cargo data must not determine whether outputs fit.
2736        std::fs::create_dir_all(
2737            first.candidates[0]
2738                .worktree
2739                .join("target/debug/incremental"),
2740        )
2741        .unwrap();
2742        std::fs::write(
2743            first.candidates[0]
2744                .worktree
2745                .join("target/debug/incremental/cache"),
2746            [0; 1000],
2747        )
2748        .unwrap();
2749        let mut disk = Fx::disk();
2750        disk.worktree_reclaim_bytes = 0;
2751        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2752        disk.worktree_reclaim_bytes = 300;
2753        let out = sweep(&f.home, &disk, Fx::now()).await;
2754        assert_eq!(out.freed_bytes, 200, "{out:?}");
2755        assert!(
2756            !first.candidates[0]
2757                .worktree
2758                .join("target/debug/output")
2759                .exists()
2760        );
2761        assert!(
2762            second.candidates[0]
2763                .worktree
2764                .join("target/debug/output")
2765                .exists()
2766        );
2767        assert!(
2768            out.retained
2769                .iter()
2770                .any(|r| r.contains("declared outputs exceed"))
2771        );
2772    }
2773
2774    #[tokio::test]
2775    async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2776        let f = Fx::new().await;
2777        let mut protected = f.run("bd00", true).await;
2778        let later = f.run("bd01", true).await;
2779        for state in [&protected, &later] {
2780            Fx::target(&state.candidates[0].worktree);
2781        }
2782        let mut disk = Fx::disk();
2783        disk.worktree_reclaim_bytes = 100;
2784        disk.fold_grace_secs = 3600;
2785        let out = sweep(&f.home, &disk, Fx::now()).await;
2786        assert_eq!(out.freed_bytes, 0);
2787        assert!(
2788            out.retained
2789                .iter()
2790                .any(|r| r.contains("within grace period"))
2791        );
2792        disk.fold_grace_secs = 0;
2793        protected.driver_exited = false;
2794        protected.driver_pid = Some(std::process::id());
2795        protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2796        protected
2797            .save_under_at(&f.home, protected.updated_at)
2798            .unwrap();
2799        let out = sweep(&f.home, &disk, Fx::now()).await;
2800        assert!(
2801            out.retained
2802                .iter()
2803                .any(|r| r.contains("run live or ownership unknown"))
2804        );
2805        assert_eq!(
2806            out.freed_bytes, 200,
2807            "live target must not consume exception: {out:?}"
2808        );
2809        let output = protected.candidates[0].worktree.join("target/debug/output");
2810        assert!(output.exists());
2811        protected.driver_exited = true;
2812        protected.save_under(&f.home).unwrap();
2813        let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2814        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2815        assert!(output.exists());
2816        drop(owner);
2817        let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2818        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2819        assert!(output.exists());
2820        drop(cargo);
2821        let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2822        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2823        assert!(output.exists());
2824        drop(lease);
2825        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2826        assert!(!output.exists());
2827    }
2828
2829    #[tokio::test]
2830    async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2831        for remove_root in [true, false] {
2832            let f = Fx::new().await;
2833            let stale = Fx::new().await;
2834            let mut missing = stale.run("aa00", false).await;
2835            Fx::target(&missing.candidates[0].worktree);
2836            missing.save_under(&f.home).unwrap();
2837            if remove_root {
2838                std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2839                // An existing non-repository proves the absent root is checked
2840                // before querying Git, even when no registrations are cached.
2841                std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2842            } else {
2843                std::fs::remove_dir_all(&stale.repo).unwrap();
2844            }
2845            let healthy = f.run("aa01", false).await;
2846            Fx::target(&healthy.candidates[0].worktree);
2847
2848            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2849            assert!(report.failures.is_empty(), "{report:?}");
2850            assert_eq!(report.freed_bytes, 200, "{report:?}");
2851            assert!(
2852                !healthy.candidates[0]
2853                    .worktree
2854                    .join("target/debug/output")
2855                    .exists()
2856            );
2857            if !remove_root {
2858                // Without the repository, ownership cannot be validated.
2859                assert!(
2860                    missing.candidates[0]
2861                        .worktree
2862                        .join("target/debug/output")
2863                        .exists()
2864                );
2865            }
2866        }
2867    }
2868
2869    #[tokio::test]
2870    async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2871        let f = Fx::new().await;
2872        let mut missing = f.run("aa00", false).await;
2873        Fx::target(&missing.candidates[0].worktree);
2874        missing.repo = missing.candidates[0].worktree.clone();
2875        missing.save_under(&f.home).unwrap();
2876        std::fs::remove_dir_all(&missing.repo).unwrap();
2877        assert!(missing.worktree_root().is_dir());
2878        assert!(f.repo.is_dir());
2879
2880        let healthy = f.run("aa01", false).await;
2881        Fx::target(&healthy.candidates[0].worktree);
2882
2883        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2884        assert!(report.failures.is_empty(), "{report:?}");
2885        assert_eq!(report.freed_bytes, 200, "{report:?}");
2886        assert!(
2887            !healthy.candidates[0]
2888                .worktree
2889                .join("target/debug/output")
2890                .exists()
2891        );
2892    }
2893
2894    #[tokio::test]
2895    async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2896        let f = Fx::new().await;
2897        let broken = Fx::new().await;
2898        let mut invalid = broken.run("aa00", false).await;
2899        Fx::target(&invalid.candidates[0].worktree);
2900        invalid.save_under(&f.home).unwrap();
2901        std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2902        let healthy = f.run("aa01", false).await;
2903        Fx::target(&healthy.candidates[0].worktree);
2904
2905        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2906        assert_eq!(report.failures.len(), 1, "{report:?}");
2907        assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2908        assert_eq!(report.freed_bytes, 200, "{report:?}");
2909        assert!(
2910            !healthy.candidates[0]
2911                .worktree
2912                .join("target/debug/output")
2913                .exists()
2914        );
2915        assert!(
2916            invalid.candidates[0]
2917                .worktree
2918                .join("target/debug/output")
2919                .exists()
2920        );
2921    }
2922
2923    async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2924        let path = reconcile::run(&f.home, &task.id, &source.id, None)
2925            .await
2926            .unwrap();
2927        serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2928    }
2929
2930    #[tokio::test]
2931    async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
2932        use crate::{
2933            chat_control::{Operation, dispatch},
2934            talk::{self, Talks},
2935        };
2936        let f = Fx::new().await;
2937        let source = f.run("ae00", true).await;
2938        let clean = f.run("ae01", false).await;
2939        let queue = crate::queue::Queue::at(f.home.join("queue"));
2940        let talks = Talks::at(f.home.join("talks"));
2941        let open_talk = |id: &str| -> talk::Talk {
2942            let mut conversation = serde_json::from_value(serde_json::json!({
2943                "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
2944                "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
2945                "seat":crate::agent::SeatState::new("talk","worker",1)
2946            }))
2947            .unwrap();
2948            talks.put(&mut conversation).unwrap();
2949            conversation
2950        };
2951        let conversation = open_talk("talk-owner");
2952        let other = open_talk("talk-other");
2953        // A task this chat did not file: the owner's request covers it too.
2954        let task = f.task(&[source.clone(), clean.clone()]);
2955        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2956        let operation = |apply: Option<&str>| Operation::ReconcileRetention {
2957            task: task.id.clone(),
2958            authority: source.id.clone(),
2959            apply: apply.map(str::to_owned),
2960        };
2961        // A plan saved outside the chat is not applied through it.
2962        let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
2963            .await
2964            .unwrap();
2965        let legacy: serde_json::Value =
2966            serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
2967        let legacy_id = legacy["id"].as_str().unwrap();
2968        assert!(
2969            dispatch(
2970                &queue,
2971                &f.home,
2972                &talks,
2973                &conversation.id,
2974                operation(Some(legacy_id))
2975            )
2976            .await
2977            .is_err()
2978        );
2979        let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
2980            .await
2981            .unwrap();
2982        assert!(summary.contains("Complete review summary"));
2983        assert!(summary.contains(&source.id) && summary.contains(&clean.id));
2984        let root = f.home.join("retention-reconciliations");
2985        let displayed: serde_json::Value =
2986            serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
2987        let id = displayed["plan"].as_str().unwrap();
2988        let dir = root.join(id);
2989        let plan: serde_json::Value =
2990            serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
2991        assert_eq!(plan["version"], 2);
2992        assert!(clean.candidates[0].worktree.exists());
2993        assert!(!dir.join("approved.json").exists());
2994        // The plan stays bound to the conversation it was saved for.
2995        assert!(
2996            dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
2997                .await
2998                .is_err()
2999        );
3000        assert!(!dir.join("approved.json").exists());
3001        // No owner phrase or quote: the request itself applies the plan. An
3002        // interruption after the durable fence keeps the audit for a retry.
3003        let error = reconcile::interrupt_chat_after_fence(
3004            &f.home,
3005            &task,
3006            id,
3007            reconcile::ChatRequest {
3008                talks: &talks,
3009                conversation: &conversation.id,
3010            },
3011        )
3012        .await
3013        .unwrap_err();
3014        assert!(
3015            error.to_string().contains("simulated interruption"),
3016            "{error:#}"
3017        );
3018        assert!(clean.candidates[0].worktree.exists());
3019        assert!(dir.join("approved.json").exists());
3020        dispatch(
3021            &queue,
3022            &f.home,
3023            &talks,
3024            &conversation.id,
3025            operation(Some(id)),
3026        )
3027        .await
3028        .unwrap();
3029        assert!(!clean.candidates[0].worktree.exists());
3030        assert!(source.candidates[0].worktree.exists());
3031        let audit: serde_json::Value =
3032            serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3033        assert_eq!(audit["version"], 2);
3034        assert_eq!(
3035            audit["owner"],
3036            serde_json::json!({"conversation": conversation.id})
3037        );
3038        dispatch(
3039            &queue,
3040            &f.home,
3041            &talks,
3042            &conversation.id,
3043            operation(Some(id)),
3044        )
3045        .await
3046        .unwrap();
3047    }
3048
3049    #[tokio::test]
3050    async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3051        let f = Fx::new().await;
3052        let source = f.run("af00", true).await;
3053        let mut clean = f.run("af01", false).await;
3054        let mut dirty = f.run("af02", false).await;
3055        for state in [&mut clean, &mut dirty] {
3056            state.status = RunStatus::Prep;
3057            state.prep_complete = Some(false);
3058            state.candidates[0].empty = false;
3059            state.save_under(&f.home).unwrap();
3060        }
3061        std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3062        let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3063        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3064        let plan = reconciliation_plan(&f, &task, &source).await;
3065        let selected: Vec<_> = plan["candidates"]
3066            .as_array()
3067            .unwrap()
3068            .iter()
3069            .filter(|e| e["selected"] == true)
3070            .collect();
3071        assert_eq!(selected.len(), 1, "{plan:#}");
3072        assert_eq!(selected[0]["run"], clean.id);
3073        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3074            .await
3075            .unwrap();
3076        assert!(!clean.candidates[0].worktree.exists());
3077        assert!(dirty.candidates[0].worktree.exists());
3078        assert!(
3079            RunState::load_under(&dirty.id, &f.home)
3080                .unwrap()
3081                .retired_to
3082                .is_none()
3083        );
3084    }
3085
3086    #[tokio::test]
3087    async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3088        let f = Fx::new().await;
3089        let mut source = f.run("aa00", true).await;
3090        let alternative = f.run("aa01", true).await;
3091        let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3092            .await
3093            .unwrap();
3094        source.reviewed_commits = Some(vec![head.clone()]);
3095        source.save_under(&f.home).unwrap();
3096        let mut runs = vec![source.clone(), alternative.clone()];
3097        for i in 2..14 {
3098            runs.push(f.run(&format!("aa{i:02}"), false).await);
3099        }
3100        let mut task = f.task(&runs);
3101        task.pr_recovery = Some(crate::pr_recovery::Selection {
3102            url: "https://example.invalid/pull/1".into(),
3103            branch: "selected".into(),
3104            head: head.clone(),
3105            source_head: Some(head.clone()),
3106            source_run: Some(source.id.clone()),
3107            source_proof: None,
3108            disposition: Some(crate::pr_recovery::Disposition::Land),
3109            instruction: task.instruction.clone(),
3110            approved_at: Timestamp::now(),
3111        });
3112        source.pr_recovery = task.pr_recovery.clone();
3113        source.save_under(&f.home).unwrap();
3114        runs[0] = source.clone();
3115        crate::queue::Queue::at(f.home.join("queue"))
3116            .put(&mut task)
3117            .unwrap();
3118        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3119        let task_before = serde_json::to_value(&task).unwrap();
3120        let mut report = Report::default();
3121        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3122        assert_eq!(
3123            report.retired, 0,
3124            "automatic cleanup must still refuse blanket markers"
3125        );
3126        assert!(
3127            report
3128                .retained
3129                .iter()
3130                .any(|reason| reason.contains("explicitly retained"))
3131        );
3132        let mut total = 0;
3133        for expected in [5, 5, 2, 0] {
3134            let plan = reconciliation_plan(&f, &task, &source).await;
3135            let selected: Vec<_> = plan["candidates"]
3136                .as_array()
3137                .unwrap()
3138                .iter()
3139                .filter(|e| e["selected"] == true)
3140                .collect();
3141            assert_eq!(selected.len(), expected, "{plan:#}");
3142            for e in &selected {
3143                assert!(
3144                    RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3145                        .unwrap()
3146                        .retired_to
3147                        .is_none()
3148                );
3149            }
3150            let operation = plan["id"].as_str().unwrap();
3151            for _ in 0..2 {
3152                reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3153                    .await
3154                    .unwrap();
3155            }
3156            total += expected;
3157        }
3158        assert_eq!(total, 12);
3159        for run in &runs {
3160            let dir = f.home.join("runs").join(&run.id);
3161            assert!(dir.join("run.json").exists());
3162            assert!(dir.join("retain-recovery-sources").exists());
3163            assert!(
3164                git::branch_exists(&f.repo, &run.candidates[0].branch)
3165                    .await
3166                    .unwrap()
3167            );
3168        }
3169        assert!(source.candidates[0].worktree.exists());
3170        assert!(alternative.candidates[0].worktree.exists());
3171        assert_eq!(
3172            git::rev_parse(&f.repo, &source.candidates[0].branch)
3173                .await
3174                .unwrap(),
3175            head
3176        );
3177        assert_eq!(
3178            serde_json::to_value(
3179                crate::queue::Queue::at(f.home.join("queue"))
3180                    .get(&task.id)
3181                    .unwrap()
3182            )
3183            .unwrap(),
3184            task_before
3185        );
3186        assert_eq!(
3187            runs.iter()
3188                .filter(|r| RunState::load_under(&r.id, &f.home)
3189                    .unwrap()
3190                    .retired_to
3191                    .is_none())
3192                .count(),
3193            2
3194        );
3195    }
3196
3197    #[tokio::test]
3198    async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3199        let f = Fx::new().await;
3200        let source = f.run("ab00", true).await;
3201        let mut runs = vec![source.clone()];
3202        for i in 1..9 {
3203            runs.push(f.run(&format!("ab{i:02}"), false).await);
3204        }
3205        std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3206        std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3207        std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3208        std::fs::write(
3209            runs[3].candidates[0].worktree.join("scratch/source"),
3210            "ignored source",
3211        )
3212        .unwrap();
3213        std::fs::write(
3214            runs[4].worktree_root().join("unknown"),
3215            "unregistered source",
3216        )
3217        .unwrap();
3218        let checkpoint = git::git(
3219            &runs[5].candidates[0].worktree,
3220            &[
3221                "rev-parse",
3222                "--path-format=absolute",
3223                "--git-path",
3224                "sequencer",
3225            ],
3226        )
3227        .await
3228        .unwrap();
3229        std::fs::create_dir(checkpoint.trim()).unwrap();
3230        runs[6].driver_exited = false;
3231        runs[6].driver_pid = Some(std::process::id());
3232        runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3233        runs[6].save_under(&f.home).unwrap();
3234        runs[7].driver_exited = false;
3235        runs[7].driver_pid = None;
3236        runs[7].driver_claimed = None; // Legacy record without first-claim evidence.
3237        runs[7].save_under(&f.home).unwrap();
3238        assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3239
3240        let hidden = &runs[8].candidates[0].worktree;
3241        git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3242            .await
3243            .unwrap();
3244        std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3245        assert!(
3246            git::git(hidden, &["status", "--porcelain"])
3247                .await
3248                .unwrap()
3249                .is_empty()
3250        );
3251        let task = f.task(&runs);
3252        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3253        let plan = reconciliation_plan(&f, &task, &source).await;
3254        assert!(
3255            plan["candidates"]
3256                .as_array()
3257                .unwrap()
3258                .iter()
3259                .all(|e| e["selected"] == false),
3260            "{plan:#}"
3261        );
3262        for run in &runs {
3263            assert!(run.candidates[0].worktree.exists());
3264        }
3265    }
3266
3267    #[tokio::test]
3268    async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3269        let f = Fx::new().await;
3270        let source = f.run("ac00", true).await;
3271        let empty = f.run("ac01", false).await;
3272        let task = f.task(&[source.clone(), empty.clone()]);
3273        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3274        let plan = reconciliation_plan(&f, &task, &source).await;
3275        std::fs::write(
3276            f.home.join("runs").join(&source.id).join("new-report"),
3277            "new review checkpoint",
3278        )
3279        .unwrap();
3280        assert!(
3281            reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3282                .await
3283                .is_err()
3284        );
3285        assert!(empty.candidates[0].worktree.exists());
3286        assert!(
3287            RunState::load_under(&empty.id, &f.home)
3288                .unwrap()
3289                .retired_to
3290                .is_none()
3291        );
3292        let marker = f
3293            .home
3294            .join("runs")
3295            .join(&empty.id)
3296            .join("retain-recovery-sources");
3297        std::fs::write(&marker, "different owner").unwrap();
3298        let plan = reconciliation_plan(&f, &task, &source).await;
3299        assert!(
3300            plan["candidates"]
3301                .as_array()
3302                .unwrap()
3303                .iter()
3304                .all(|e| e["selected"] == false)
3305        );
3306    }
3307
3308    #[tokio::test]
3309    async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3310        for partial in [false, true] {
3311            let f = Fx::new().await;
3312            let source = f.run("ab10", true).await;
3313            let first = f.run("ab11", false).await;
3314            let second = f.run("ab12", false).await;
3315            let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3316            crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3317            let plan = reconciliation_plan(&f, &task, &source).await;
3318            let id = plan["id"].as_str().unwrap();
3319            let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3320            // Matching bytes without this operation's fence are not trusted.
3321            std::fs::write(&staging, b"").unwrap();
3322            assert!(
3323                reconcile::run(&f.home, &task.id, &source.id, Some(id))
3324                    .await
3325                    .is_err()
3326            );
3327            assert!(staging.exists());
3328            std::fs::remove_file(&staging).unwrap();
3329            let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3330                .await
3331                .unwrap_err();
3332            assert!(
3333                error.to_string().contains("before run state rename"),
3334                "{error:#}"
3335            );
3336            assert!(
3337                RunState::load_under(&first.id, &f.home)
3338                    .unwrap()
3339                    .retired_to
3340                    .is_some()
3341            );
3342            assert!(
3343                RunState::load_under(&second.id, &f.home)
3344                    .unwrap()
3345                    .retired_to
3346                    .is_none()
3347            );
3348            assert!(first.candidates[0].worktree.exists());
3349            assert!(second.candidates[0].worktree.exists());
3350            let interrupted = std::fs::read(&staging).unwrap();
3351            // An unrelated file must survive a refused retry unchanged.
3352            std::fs::write(&staging, b"unrelated recovery source").unwrap();
3353            assert!(
3354                reconcile::run(&f.home, &task.id, &source.id, Some(id))
3355                    .await
3356                    .is_err()
3357            );
3358            assert_eq!(
3359                std::fs::read(&staging).unwrap(),
3360                b"unrelated recovery source"
3361            );
3362            std::fs::write(&staging, interrupted).unwrap();
3363            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3364                .await
3365                .unwrap();
3366            assert!(!staging.exists());
3367            assert!(!first.candidates[0].worktree.exists());
3368            assert!(!second.candidates[0].worktree.exists());
3369            assert_eq!(
3370                RunState::load_under(&second.id, &f.home)
3371                    .unwrap()
3372                    .retired_to,
3373                Some(source.id)
3374            );
3375        }
3376    }
3377
3378    #[tokio::test]
3379    async fn reconciliation_resumes_only_its_exact_durable_fence() {
3380        let f = Fx::new().await;
3381        let source = f.run("ad00", true).await;
3382        let empty = f.run("ad01", false).await;
3383        let task = f.task(&[source.clone(), empty.clone()]);
3384        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3385        let plan = reconciliation_plan(&f, &task, &source).await;
3386        let id = plan["id"].as_str().unwrap();
3387        assert!(
3388            reconcile::interrupt_after_fence(&f.home, &task, id)
3389                .await
3390                .is_err()
3391        );
3392        assert!(empty.candidates[0].worktree.exists());
3393        assert_eq!(
3394            RunState::load_under(&empty.id, &f.home)
3395                .unwrap()
3396                .retired_to
3397                .as_ref(),
3398            Some(&source.id)
3399        );
3400        reconcile::run(&f.home, &task.id, &source.id, Some(id))
3401            .await
3402            .unwrap();
3403        assert!(!empty.candidates[0].worktree.exists());
3404        // Model an interrupted removal: restore precisely the approved base worktree.
3405        git::worktree_add_branch(
3406            &f.repo,
3407            &empty.candidates[0].worktree,
3408            "restore-empty",
3409            &f.base,
3410        )
3411        .await
3412        .unwrap();
3413        // Recreated Git administration differs from approval and is refused.
3414        assert!(
3415            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3416                .await
3417                .is_err()
3418        );
3419        assert!(empty.candidates[0].worktree.exists());
3420        let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3421        changed.prep_failure = Some("new owner edit".into());
3422        changed.save_under(&f.home).unwrap();
3423        assert!(
3424            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3425                .await
3426                .is_err()
3427        );
3428    }
3429
3430    #[tokio::test]
3431    async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3432        let f = Fx::new().await;
3433        let source = f.run("ae00", true).await;
3434        let mut prep = f.run("ae01", false).await;
3435        let candidate = prep.candidates.pop().unwrap();
3436        prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3437        prep.save_under(&f.home).unwrap();
3438        let reset = f.run("ae02", false).await;
3439        let wt = &reset.candidates[0].worktree;
3440        std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3441        git::git(wt, &["add", "."]).await.unwrap();
3442        git::git(wt, &["commit", "-m", "unique implementation"])
3443            .await
3444            .unwrap();
3445        git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3446        let unknown = f.run("ae03", false).await;
3447        let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3448        let mut raw: serde_json::Value =
3449            serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3450        raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3451        std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3452        let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3453        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3454        let plan = reconciliation_plan(&f, &task, &source).await;
3455        let selected: Vec<_> = plan["candidates"]
3456            .as_array()
3457            .unwrap()
3458            .iter()
3459            .filter(|e| e["selected"] == true)
3460            .collect();
3461        assert_eq!(selected.len(), 1, "{plan:#}");
3462        assert_eq!(selected[0]["run"], prep.id);
3463        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3464            .await
3465            .unwrap();
3466        assert!(!candidate.worktree.exists());
3467        assert!(reset.candidates[0].worktree.exists());
3468        assert!(unknown.candidates[0].worktree.exists());
3469    }
3470
3471    #[tokio::test]
3472    async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3473        let f = Fx::new().await;
3474        let mut state = f.run("aaaa", true).await;
3475        let candidate = state.candidates[0].worktree.clone();
3476        std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3477        Fx::target(&candidate);
3478        for seat in ["review-1", "review-2"] {
3479            let wt = state.worktree_root().join(seat);
3480            git::worktree_add_detached(&f.repo, &wt, &f.base)
3481                .await
3482                .unwrap();
3483            Fx::target(&wt);
3484        }
3485        state.status = RunStatus::Landing;
3486        state.save_under(&f.home).unwrap();
3487        let second = Fx::new().await;
3488        let other = second.run("bbbb", true).await;
3489        // A global store may contain repositories with different worktree roots.
3490        let mut imported = other.clone();
3491        imported.save_under(&f.home).unwrap();
3492        Fx::target(&other.candidates[0].worktree);
3493        let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3494        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3495        assert!(
3496            candidate.join("target/debug/output").exists(),
3497            "active writer protected"
3498        );
3499        assert!(
3500            !state
3501                .worktree_root()
3502                .join("review-1/target/debug/output")
3503                .exists()
3504        );
3505        assert!(
3506            !state
3507                .worktree_root()
3508                .join("review-2/target/debug/output")
3509                .exists()
3510        );
3511        assert!(
3512            !other.candidates[0]
3513                .worktree
3514                .join("target/debug/output")
3515                .exists()
3516        );
3517        assert!(out.freed_bytes >= 600);
3518        drop(guard);
3519        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3520        assert!(out.freed_bytes >= 200);
3521        assert_eq!(
3522            std::fs::read_to_string(candidate.join("source")).unwrap(),
3523            "uncommitted work"
3524        );
3525        assert!(
3526            git::branch_exists(&f.repo, &state.candidates[0].branch)
3527                .await
3528                .unwrap()
3529        );
3530    }
3531
3532    #[tokio::test]
3533    async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3534        let f = Fx::new().await;
3535        let state = f.run("cc01", false).await;
3536        let wt = &state.candidates[0].worktree;
3537        Fx::target(wt);
3538        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3539            let dir = wt.join("target").join(dir);
3540            std::fs::create_dir_all(&dir).unwrap();
3541            std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3542        }
3543        // A dep-info dependency is an input, never deletion authority.
3544        let manifest = wt.join("target/debug/output.d");
3545        let output = wt.join("target/debug/output");
3546        std::fs::write(
3547            &manifest,
3548            format!(
3549                "{}: {}\n",
3550                output.display(),
3551                wt.join("target/debug/notes.txt").display()
3552            ),
3553        )
3554        .unwrap();
3555        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3556        assert_eq!(report.freed_bytes, 200);
3557        assert!(!output.exists());
3558        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3559            assert_eq!(
3560                std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3561                "user evidence"
3562            );
3563        }
3564        // Metadata remains available for inspection and future safe rebuilds.
3565        assert!(manifest.exists());
3566        assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3567        std::fs::write(&output, "user note replacing an old binary").unwrap();
3568        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3569        assert_eq!(report.freed_bytes, 0);
3570        assert_eq!(
3571            std::fs::read_to_string(&output).unwrap(),
3572            "user note replacing an old binary"
3573        );
3574    }
3575
3576    #[tokio::test]
3577    async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3578        let f = Fx::new().await;
3579        let state = f.run("cc02", false).await;
3580        let wt = &state.candidates[0].worktree;
3581        Fx::target(wt);
3582        let mut disk = Fx::disk();
3583        disk.worktree_limit_bytes = u64::MAX;
3584        let now = Fx::now();
3585        let first = sweep(&f.home, &disk, now).await;
3586        std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3587        assert_eq!(
3588            sweep(&f.home, &disk, now).await.measured_bytes,
3589            first.measured_bytes
3590        );
3591        assert!(
3592            sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3593                .await
3594                .measured_bytes
3595                > first.measured_bytes
3596        );
3597        disk.worktree_limit_bytes = 0;
3598        assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3599    }
3600
3601    #[tokio::test]
3602    async fn unfinished_source_fold_reports_its_retention_reason() {
3603        let f = Fx::new().await;
3604        let mut source = f.run("cc03", true).await;
3605        let task = f.task(&[source.clone()]);
3606        let error = crate::graph::fold_run(&mut source, true, &f.home)
3607            .await
3608            .unwrap_err();
3609        assert!(error.to_string().contains(&task.id));
3610        assert!(source.candidates[0].worktree.exists());
3611        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3612        let error = crate::graph::fold_run(&mut source, true, &f.home)
3613            .await
3614            .unwrap_err();
3615        assert!(error.to_string().contains("explicitly retained"));
3616    }
3617
3618    #[tokio::test]
3619    async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3620        let f = Fx::new().await;
3621        let mut source = f.run("cc04", true).await;
3622        let mut task = f.task(&[source.clone()]);
3623        task.hold_manual(Some("queue settlement pending".into()));
3624        let queue = crate::queue::Queue::at(f.home.join("queue"));
3625        queue.put(&mut task).unwrap();
3626        source.merge = Some(crate::run::MergeOutcome {
3627            mode: crate::config::MergeMode::Pr,
3628            ok: true,
3629            detail: "confirmed merged".into(),
3630            empty: false,
3631        });
3632        // Publication success and terminal labels alone are insufficient.
3633        for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3634            source.status = status;
3635            assert!(
3636                crate::graph::fold_run(&mut source, true, &f.home)
3637                    .await
3638                    .is_err()
3639            );
3640            assert!(source.candidates[0].worktree.exists());
3641        }
3642        git::git(
3643            &f.repo,
3644            &["merge", "--ff-only", &source.candidates[0].branch],
3645        )
3646        .await
3647        .unwrap();
3648        git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3649            .await
3650            .unwrap();
3651        source.status = RunStatus::Merged;
3652        source.save_under(&f.home).unwrap();
3653        assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3654        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3655        assert!(
3656            crate::graph::fold_run(&mut source, true, &f.home)
3657                .await
3658                .unwrap_err()
3659                .to_string()
3660                .contains("explicitly retained")
3661        );
3662        // The fixture's explicit marker is removed only to exercise ordinary
3663        // confirmed-merge cleanup; no production cleanup removes such markers.
3664        std::fs::remove_file(
3665            f.home
3666                .join("runs")
3667                .join(&source.id)
3668                .join("retain-recovery-sources"),
3669        )
3670        .unwrap();
3671        let (folded, unreadable) = crate::clean::fold_due(
3672            &f.home.join("runs"),
3673            &f.home,
3674            &f.bay,
3675            &Fx::disk(),
3676            Fx::now(),
3677        )
3678        .await
3679        .unwrap();
3680        assert_eq!((folded, unreadable), (1, 0));
3681        assert!(!source.candidates[0].worktree.exists());
3682        let evidence = f
3683            .home
3684            .join("runs")
3685            .join(&source.id)
3686            .join("artifacts/fold-evidence");
3687        let names: Vec<_> = std::fs::read_dir(evidence)
3688            .unwrap()
3689            .map(|entry| entry.unwrap().file_name())
3690            .collect();
3691        assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3692        assert_eq!(
3693            queue.get(&task.id).unwrap().status,
3694            crate::queue::TaskStatus::Held
3695        );
3696    }
3697
3698    #[tokio::test]
3699    async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3700        for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3701            let f = Fx::new().await;
3702            let mut state = f.run("cc06", true).await;
3703            let candidate = state.candidates[0].clone();
3704            let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3705            std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3706            git::rescue_commit(&candidate.worktree, "second candidate change")
3707                .await
3708                .unwrap();
3709            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3710            state.tally = Some(crate::run::Tally {
3711                first_choice: Default::default(),
3712                borda: Default::default(),
3713                winner: 'A',
3714                rankings: 0,
3715                unanimous_initial: true,
3716                deliberated: false,
3717                changed_votes: 0,
3718                unanimous_final: true,
3719                tie_break: None,
3720                judges: 0,
3721                present: 0,
3722                quorum: 0,
3723                met_quorum: true,
3724                uncontested: Some("fixture".into()),
3725            });
3726            // A reviewer pinned to an intermediate candidate commit must also
3727            // be reclaimable, although neither commit ID lands in the squash.
3728            let reviewer = state.worktree_root().join("review-1");
3729            git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3730                .await
3731                .unwrap();
3732            // Concurrent base changes and later base advances prevent a mere
3733            // equality check against the current base tree from passing.
3734            std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3735            git::rescue_commit(&f.repo, "advance before squash")
3736                .await
3737                .unwrap();
3738            let published = if variant == "partial" {
3739                &intermediate
3740            } else {
3741                &head
3742            };
3743            git::git(&f.repo, &["merge", "--squash", published])
3744                .await
3745                .unwrap();
3746            git::git(&f.repo, &["commit", "-m", "squash candidate"])
3747                .await
3748                .unwrap();
3749            let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3750            assert!(
3751                !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3752                    .await
3753                    .unwrap()
3754                    .ok()
3755            );
3756            std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3757            git::rescue_commit(&f.repo, "advance after squash")
3758                .await
3759                .unwrap();
3760            git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3761                .await
3762                .unwrap();
3763            if matches!(variant, "checkpoint" | "reviewer") {
3764                // A reset hides unique work from HEAD, but not its reflog.
3765                std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3766                git::rescue_commit(&reviewer, "unpublished checkpoint")
3767                    .await
3768                    .unwrap();
3769                if variant == "checkpoint" {
3770                    git::git(&reviewer, &["reset", "--hard", &intermediate])
3771                        .await
3772                        .unwrap();
3773                }
3774            }
3775            if variant == "loser" {
3776                let mut loser = candidate.clone();
3777                loser.label = 'B';
3778                loser.branch = "magi/cc06/B".into();
3779                loser.worktree = state.worktree_root().join("cand-B");
3780                git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3781                    .await
3782                    .unwrap();
3783                std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3784                git::rescue_commit(&loser.worktree, "independent loser")
3785                    .await
3786                    .unwrap();
3787                state.candidates.push(loser);
3788            }
3789            state.status = RunStatus::Merged;
3790            state.merge = Some(crate::run::MergeOutcome {
3791                mode: crate::config::MergeMode::Pr,
3792                ok: true,
3793                detail: "candidate A squash merged".into(),
3794                empty: false,
3795            });
3796            state.save_under(&f.home).unwrap();
3797            let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3798            let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3799            result.unwrap();
3800            assert!(
3801                !candidate.worktree.exists() && !reviewer.exists(),
3802                "{variant}"
3803            );
3804            assert!(
3805                !git::branch_exists(&f.repo, &candidate.branch)
3806                    .await
3807                    .unwrap()
3808            );
3809            let artifacts = f
3810                .home
3811                .join("runs")
3812                .join(&state.id)
3813                .join("artifacts/fold-evidence");
3814            assert_eq!(
3815                artifacts.join(format!("{head}.patch")).exists(),
3816                variant == "partial",
3817                "only a partially published winner needs evidence: {variant}"
3818            );
3819            assert_eq!(
3820                artifacts.join(format!("{intermediate}.patch")).exists(),
3821                variant == "partial",
3822                "squash proof also covers intermediate reviewer tips: {variant}"
3823            );
3824            let patches: Vec<_> = std::fs::read_dir(&artifacts)
3825                .unwrap()
3826                .map(|entry| entry.unwrap().path())
3827                .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3828                .collect();
3829            let expected = match variant {
3830                "partial" => 2,
3831                "loser" | "reviewer" => 1,
3832                _ => 0,
3833            };
3834            assert_eq!(patches.len(), expected, "{variant}");
3835            if variant == "loser" {
3836                assert!(
3837                    std::fs::read_to_string(&patches[0])
3838                        .unwrap()
3839                        .contains("losing candidate")
3840                );
3841            }
3842            assert!(artifacts.join("manifest.json").exists());
3843            if variant == "reviewer" {
3844                assert!(
3845                    std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3846                        .unwrap()
3847                        .contains("reviewer checkpoint")
3848                );
3849            }
3850            crate::graph::fold_run(&mut state, true, &f.home)
3851                .await
3852                .unwrap();
3853        }
3854    }
3855
3856    #[tokio::test]
3857    async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3858        for missing in [true, false] {
3859            let f = Fx::new().await;
3860            let mut state = f.run("cc05", true).await;
3861            // A fetched base exists, but these local review changes were not
3862            // incorporated by ancestry, so their recovery evidence is required.
3863            git::git(
3864                &f.repo,
3865                &["update-ref", "refs/remotes/origin/main", &f.base],
3866            )
3867            .await
3868            .unwrap();
3869            let candidate = state.candidates[0].clone();
3870            if !missing {
3871                std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3872                git::rescue_commit(&f.repo, "advance base before rebase")
3873                    .await
3874                    .unwrap();
3875                git::git(&candidate.worktree, &["rebase", "main"])
3876                    .await
3877                    .unwrap();
3878                // Match the runner's post-rebase synchronization, which
3879                // clears the merge backend's transient AUTO_MERGE checkpoint.
3880                git::sync_to_head(&candidate.worktree).await.unwrap();
3881            }
3882            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3883            // Model review-only state: no implementation-time snapshot exists.
3884            state.reviewed_commits = Some(Vec::new());
3885            state.candidates[0].worktree = state.worktree_root().join("under-review");
3886            git::git(
3887                &f.repo,
3888                &[
3889                    "worktree",
3890                    "move",
3891                    candidate.worktree.to_str().unwrap(),
3892                    state.candidates[0].worktree.to_str().unwrap(),
3893                ],
3894            )
3895            .await
3896            .unwrap();
3897            let reviewer = state.worktree_root().join("review-1");
3898            git::worktree_add_detached(&f.repo, &reviewer, &head)
3899                .await
3900                .unwrap();
3901            // A reset/rebase can leave unpublished recovery history. It must
3902            // not keep any remaining owned worktree alive indefinitely.
3903            std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3904            git::rescue_commit(&reviewer, "old review history")
3905                .await
3906                .unwrap();
3907            git::git(&reviewer, &["reset", "--hard", &head])
3908                .await
3909                .unwrap();
3910            if missing {
3911                std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3912            }
3913            let run_dir = f.home.join("runs").join(&state.id);
3914            let artifacts = run_dir.join("artifacts");
3915            std::fs::create_dir_all(&artifacts).unwrap();
3916            std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3917            if missing {
3918                std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3919            }
3920            state.status = RunStatus::Merged;
3921            state.merge = Some(crate::run::MergeOutcome {
3922                mode: crate::config::MergeMode::Pr,
3923                ok: true,
3924                detail: "confirmed squash merge".into(),
3925                empty: false,
3926            });
3927            state.save_under(&f.home).unwrap();
3928            // Unexpected external paths remain protected, even if absent.
3929            let owned = state.candidates[0].worktree.clone();
3930            state.candidates[0].worktree = f.home.join("missing-external");
3931            assert!(
3932                crate::graph::fold_run(&mut state, true, &f.home)
3933                    .await
3934                    .unwrap_err()
3935                    .to_string()
3936                    .contains("outside the inventoried")
3937            );
3938            state.candidates[0].worktree = owned;
3939            // A live output borrower blocks the entire preflight.
3940            let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
3941            assert!(
3942                crate::graph::fold_run(&mut state, true, &f.home)
3943                    .await
3944                    .is_err()
3945            );
3946            assert!(reviewer.exists());
3947            drop(guard);
3948            // Failure to persist evidence must not cause partial removal.
3949            let evidence = artifacts.join("fold-evidence");
3950            std::fs::write(&evidence, "blocked").unwrap();
3951            assert!(
3952                crate::graph::fold_run(&mut state, true, &f.home)
3953                    .await
3954                    .is_err()
3955            );
3956            assert!(reviewer.exists());
3957            std::fs::remove_file(&evidence).unwrap();
3958            std::fs::create_dir_all(&evidence).unwrap();
3959            let oversized = evidence.join("capacity-fixture");
3960            std::fs::File::create(&oversized)
3961                .unwrap()
3962                .set_len(64 * 1024 * 1024)
3963                .unwrap();
3964            assert!(
3965                crate::graph::fold_run(&mut state, true, &f.home)
3966                    .await
3967                    .is_err()
3968            );
3969            assert!(reviewer.exists());
3970            std::fs::remove_file(oversized).unwrap();
3971            // Ignored recovery files and unfinished Git operations still hold
3972            // all sources, even for a confirmed merge.
3973            std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
3974            std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
3975            assert!(
3976                crate::graph::fold_run(&mut state, true, &f.home)
3977                    .await
3978                    .is_err()
3979            );
3980            std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
3981            let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
3982                .await
3983                .unwrap();
3984            std::fs::write(&checkpoint, &head).unwrap();
3985            assert!(
3986                crate::graph::fold_run(&mut state, true, &f.home)
3987                    .await
3988                    .is_err()
3989            );
3990            std::fs::remove_file(checkpoint).unwrap();
3991            let (folded, unreadable) = crate::clean::fold_due(
3992                &f.home.join("runs"),
3993                &f.home,
3994                &f.bay,
3995                &Fx::disk(),
3996                Fx::now(),
3997            )
3998            .await
3999            .unwrap();
4000            if folded != 1 {
4001                panic!(
4002                    "normal cleanup failed: {:#}",
4003                    crate::graph::fold_run(&mut state, true, &f.home)
4004                        .await
4005                        .unwrap_err()
4006                );
4007            }
4008            assert_eq!((folded, unreadable), (1, 0));
4009            assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4010            state = RunState::load_under(&state.id, &f.home).unwrap();
4011            assert!(state.candidates[0].folded);
4012            let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4013            crate::graph::fold_run(&mut state, true, &f.home)
4014                .await
4015                .unwrap();
4016            assert_eq!(
4017                std::fs::read(evidence.join("manifest.json")).unwrap(),
4018                manifest
4019            );
4020            git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4021                .await
4022                .unwrap();
4023            git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4024            assert!(
4025                !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4026                    .await
4027                    .unwrap()
4028                    .ok()
4029            );
4030            // Restore the modified file into a fresh repository from artifact
4031            // bytes alone; no original commit or blob is consulted.
4032            let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4033            let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4034                .as_str()
4035                .unwrap();
4036            let recovered = f.home.join("recovered");
4037            std::fs::create_dir_all(&recovered).unwrap();
4038            git::git(&recovered, &["init"]).await.unwrap();
4039            std::fs::write(
4040                recovered.join("source"),
4041                std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4042            )
4043            .unwrap();
4044            git::git(
4045                &recovered,
4046                &[
4047                    "apply",
4048                    evidence.join(format!("{head}.patch")).to_str().unwrap(),
4049                ],
4050            )
4051            .await
4052            .unwrap();
4053            assert_eq!(
4054                std::fs::read_to_string(recovered.join("source")).unwrap(),
4055                "cc05"
4056            );
4057            assert!(
4058                !std::fs::read(evidence.join(format!("{head}.patch")))
4059                    .unwrap()
4060                    .is_empty()
4061            );
4062            assert_eq!(
4063                std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4064                "review findings"
4065            );
4066            if missing {
4067                assert_eq!(
4068                    std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4069                    "original snapshot"
4070                );
4071            } else {
4072                assert!(!artifacts.join("cand-A.patch").exists());
4073            }
4074            assert!(run_dir.join("run.json").exists());
4075        }
4076    }
4077
4078    #[tokio::test]
4079    async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4080        use std::os::unix::fs::PermissionsExt as _;
4081        let f = Fx::new().await;
4082        let source = f.run("dd03", true).await;
4083        let empty = f.run("dd04", false).await;
4084        let task = f.task(&[source, empty.clone()]);
4085        let entered = f.repo.join("proof-entered");
4086        let release = f.repo.join("proof-release");
4087        let hook = f.repo.join("proof-monitor");
4088        std::fs::write(&hook, format!(
4089            "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4090            entered.display(), release.display())).unwrap();
4091        std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4092        git::git(
4093            &f.repo,
4094            &["config", "core.fsmonitor", hook.to_str().unwrap()],
4095        )
4096        .await
4097        .unwrap();
4098        struct ReleaseProof(PathBuf);
4099        impl Drop for ReleaseProof {
4100            fn drop(&mut self) {
4101                let _ = std::fs::write(&self.0, "continue");
4102            }
4103        }
4104        let _release_on_panic = ReleaseProof(release.clone());
4105        let home = f.home.clone();
4106        let sweep = tokio::spawn(async move {
4107            let mut report = Report::default();
4108            retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4109            report
4110        });
4111        // These are deadlock watchdogs, not speed assertions. The proof stays
4112        // blocked until the task update completes, even on a loaded runner.
4113        let watchdog = std::time::Duration::from_secs(60);
4114        tokio::time::timeout(watchdog, async {
4115            while !entered.exists() {
4116                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4117            }
4118        })
4119        .await
4120        .expect("Git retirement proof did not enter the blocking hook");
4121        let queue = crate::queue::Queue::at(f.home.join("queue"));
4122        let mut selected = queue.get(&task.id).unwrap();
4123        selected.resume_override = Some(crate::queue::OperatorResume {
4124            question_id: "during-slow-proof".into(),
4125            at: Fx::now(),
4126            conductor_rehold: None,
4127            forced: false,
4128            pinned_run: Some(empty.id.clone()),
4129        });
4130        let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4131        // The proof is released only after the update returns, so correctness
4132        // is decided by ordering, not speed. The 60-second bound only detects
4133        // a writer blocked behind the proof; it is not a performance threshold.
4134        let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4135            Ok(result) => {
4136                result.unwrap().unwrap();
4137                true
4138            }
4139            Err(_) => {
4140                std::fs::write(&release, "continue").unwrap();
4141                update.await.unwrap().unwrap();
4142                false
4143            }
4144        };
4145        std::fs::write(release, "continue").unwrap();
4146        let report = tokio::time::timeout(watchdog, sweep)
4147            .await
4148            .expect("Git retirement proof did not finish after release")
4149            .unwrap();
4150        assert!(
4151            updated_while_blocked,
4152            "task writers must not wait behind Git retirement proof"
4153        );
4154        assert_eq!(report.retired, 0);
4155        assert!(report.retained.iter().any(|r| r.contains("task changed")));
4156        assert!(empty.candidates[0].worktree.exists());
4157        assert!(
4158            RunState::load_under(&empty.id, &f.home)
4159                .unwrap()
4160                .retired_to
4161                .is_none()
4162        );
4163    }
4164
4165    #[tokio::test]
4166    async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4167        let f = Fx::new().await;
4168        let source = f.run("dd01", true).await;
4169        let mut empty = f.run("dd02", false).await;
4170        let expected = f.task(&[source.clone(), empty.clone()]);
4171        let queue = crate::queue::Queue::at(f.home.join("queue"));
4172        let _claim = queue.claim(&expected.id).unwrap();
4173        // The proof holds run/output leases, never the task write lock.
4174        let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4175        let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4176        let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4177        let mut selected = queue.get(&expected.id).unwrap();
4178        selected.resume_override = Some(crate::queue::OperatorResume {
4179            question_id: "new-owner-decision".into(),
4180            at: Fx::now(),
4181            conductor_rehold: None,
4182            forced: false,
4183            pinned_run: Some(empty.id.clone()),
4184        });
4185        queue.put(&mut selected).unwrap();
4186        let error = commit_retirement(
4187            &f.home,
4188            &queue,
4189            &expected,
4190            &mut empty,
4191            &source.id,
4192            &expected.runs,
4193            Fx::now(),
4194        )
4195        .unwrap_err();
4196        assert!(error.to_string().contains("task changed"));
4197        assert!(
4198            RunState::load_under(&empty.id, &f.home)
4199                .unwrap()
4200                .retired_to
4201                .is_none()
4202        );
4203        assert!(empty.candidates[0].worktree.exists());
4204        assert_eq!(
4205            queue
4206                .get(&expected.id)
4207                .unwrap()
4208                .resume_override
4209                .unwrap()
4210                .pinned_run,
4211            Some(empty.id.clone())
4212        );
4213        assert!(
4214            !f.home
4215                .join("recovery-lineages")
4216                .join(format!("{}.json", expected.id))
4217                .exists()
4218        );
4219        // Re-reading the new snapshot must not bypass its explicit pin either.
4220        assert!(
4221            commit_retirement(
4222                &f.home,
4223                &queue,
4224                &selected,
4225                &mut empty,
4226                &source.id,
4227                &selected.runs,
4228                Fx::now()
4229            )
4230            .unwrap_err()
4231            .to_string()
4232            .contains("selected this recovery source")
4233        );
4234    }
4235
4236    #[tokio::test]
4237    async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4238        let f = Fx::new().await;
4239        let state = f.run("cccc", false).await;
4240        let wt = &state.candidates[0].worktree;
4241        Fx::target(wt);
4242        std::fs::write(wt.join("target/notes"), "user file").unwrap();
4243        std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4244        assert!(
4245            !declared_outputs(&wt.join("target"))
4246                .unwrap()
4247                .contains(&wt.join("target/debug/link"))
4248        );
4249        std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4250        let cargo = cargo_locks(&wt.join("target")).unwrap();
4251        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4252        assert_eq!(out.freed_bytes, 0);
4253        assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4254        drop(cargo);
4255        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4256        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4257        assert_eq!(out.freed_bytes, 0);
4258        drop(driver);
4259        let exclusive = OutputLease::take(wt, true).unwrap();
4260        assert!(
4261            OutputLease::borrow(wt).is_err(),
4262            "resume cannot start a writer during cleanup"
4263        );
4264        drop(exclusive);
4265        assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4266        assert_eq!(
4267            std::fs::read(wt.join("target/notes")).unwrap(),
4268            b"user file"
4269        );
4270    }
4271
4272    #[tokio::test]
4273    async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4274        for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4275            let f = Fx::new().await;
4276            let source = f.run("aa10", true).await;
4277            let empty = f.run("aa11", false).await;
4278            f.task(&[source, empty.clone()]);
4279            let wt = &empty.candidates[0].worktree;
4280            Fx::target(wt);
4281            let path = wt.join("target").join(leftover);
4282            if leftover.ends_with(".log") {
4283                std::fs::write(&path, "keep log\n").unwrap();
4284            } else {
4285                // Empty unknown directories are not present in residue_files.
4286                std::fs::create_dir(&path).unwrap();
4287            }
4288            let mut disk = Fx::disk();
4289            disk.auto_fold = true;
4290            let report = sweep(&f.home, &disk, Fx::now()).await;
4291            assert_eq!(report.freed_bytes, 200, "{report:?}");
4292            assert_eq!(report.retired, 0, "{report:?}");
4293            assert!(!wt.join("target/debug/output").exists());
4294            assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4295            assert!(path.exists());
4296            if leftover.ends_with(".log") {
4297                assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4298            }
4299            assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4300            assert!(worktrees(&empty).await.unwrap().contains(wt));
4301            assert!(
4302                git::branch_exists(&f.repo, &empty.candidates[0].branch)
4303                    .await
4304                    .unwrap()
4305            );
4306            assert!(
4307                RunState::load_under(&empty.id, &f.home)
4308                    .unwrap()
4309                    .retired_to
4310                    .is_none()
4311            );
4312        }
4313    }
4314
4315    #[tokio::test]
4316    async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4317        let f = Fx::new().await;
4318        let state = f.run("aa12", false).await;
4319        let wt = &state.candidates[0].worktree;
4320        Fx::target(wt);
4321        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4322        assert_eq!(report.freed_bytes, 200, "{report:?}");
4323        assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4324        for name in ["tmp", "magi-tools", "log"] {
4325            let path = wt.join("target").join(name);
4326            std::fs::create_dir(&path).unwrap();
4327            assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4328            std::fs::remove_dir(path).unwrap();
4329        }
4330        assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4331    }
4332
4333    #[tokio::test]
4334    async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4335        for invalid in [
4336            "tag",
4337            "identity",
4338            "tracked",
4339            "profile-file",
4340            "profile-symlink",
4341            "identity-symlink",
4342        ] {
4343            let f = Fx::new().await;
4344            let state = f.run("aa13", false).await;
4345            let wt = &state.candidates[0].worktree;
4346            Fx::target(wt);
4347            std::fs::create_dir(wt.join("target/tmp")).unwrap();
4348            let target = wt.join("target");
4349            match invalid {
4350                "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4351                "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4352                "tracked" => {
4353                    git::git(wt, &["add", "-f", "target/debug/output"])
4354                        .await
4355                        .unwrap();
4356                }
4357                "profile-file" => {
4358                    std::fs::write(target.join("release"), "keep").unwrap();
4359                }
4360                "profile-symlink" => {
4361                    std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4362                        .unwrap();
4363                }
4364                "identity-symlink" => {
4365                    std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4366                        .unwrap();
4367                    std::os::unix::fs::symlink(
4368                        target.join("saved-info"),
4369                        target.join(".rustc_info.json"),
4370                    )
4371                    .unwrap();
4372                }
4373                _ => unreachable!(),
4374            }
4375            let before = std::fs::read(target.join("debug/output")).unwrap();
4376            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4377            assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4378            assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4379            assert!(target.join("tmp").is_dir());
4380        }
4381    }
4382
4383    #[tokio::test]
4384    async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4385        let f = Fx::new().await;
4386        let source = f.run("0000", true).await;
4387        let empty = f.run("0001", false).await;
4388        let changed = f.run("0002", false).await;
4389        let unknown = f.run("0003", false).await;
4390        let unproven = f.run("0004", false).await;
4391        f.task(&[
4392            source.clone(),
4393            empty.clone(),
4394            changed.clone(),
4395            unknown.clone(),
4396            unproven.clone(),
4397        ]);
4398        for run in [&empty, &changed, &unknown] {
4399            Fx::target(&run.candidates[0].worktree);
4400        }
4401        let reviewer = empty.worktree_root().join("review-1");
4402        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4403            .await
4404            .unwrap();
4405        Fx::target(&reviewer);
4406        let mut disk = Fx::disk();
4407        disk.auto_fold = false;
4408        let report = sweep(&f.home, &disk, Fx::now()).await;
4409        assert_eq!(report.freed_bytes, 800, "{report:?}");
4410        let wt = &empty.candidates[0].worktree;
4411        assert!(wt.join("target/debug/output.d").exists());
4412        assert!(!wt.join("target/debug/output").exists());
4413        let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4414        let proof: CargoResidue =
4415            serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4416        assert_eq!(
4417            proof.files[Path::new("debug/output.d")],
4418            std::fs::read(wt.join("target/debug/output.d")).unwrap()
4419        );
4420        std::fs::write(
4421            changed.candidates[0].worktree.join("target/debug/output.d"),
4422            "owner notes",
4423        )
4424        .unwrap();
4425        std::fs::write(
4426            unknown.candidates[0].worktree.join("target/debug/notes"),
4427            "keep",
4428        )
4429        .unwrap();
4430        Fx::target(&unproven.candidates[0].worktree);
4431        std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4432        disk.auto_fold = true;
4433        let locks = cargo_locks(&wt.join("target")).unwrap();
4434        let mut report = Report::default();
4435        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4436        assert_eq!(report.retired, 0, "{report:?}");
4437        assert!(
4438            report
4439                .retained
4440                .iter()
4441                .any(|reason| reason.contains("Cargo build"))
4442        );
4443        drop(locks);
4444        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4445        assert_eq!(report.retired, 1, "{report:?}");
4446        assert!(!wt.exists());
4447        assert!(!reviewer.exists());
4448        assert!(
4449            residue_proof(&f.home, &empty.id, &reviewer)
4450                .unwrap()
4451                .exists()
4452        );
4453        assert!(
4454            proof_path.exists(),
4455            "metadata evidence outlives the worktree"
4456        );
4457        assert_eq!(
4458            RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4459            Some(source.id)
4460        );
4461        for run in [&changed, &unknown, &unproven] {
4462            assert!(run.candidates[0].worktree.exists());
4463            assert!(
4464                RunState::load_under(&run.id, &f.home)
4465                    .unwrap()
4466                    .retired_to
4467                    .is_none()
4468            );
4469        }
4470    }
4471
4472    #[tokio::test]
4473    async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4474        let f = Fx::new().await;
4475        let source = f.run("0000", true).await;
4476        let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4477            .await
4478            .unwrap();
4479        let mut runs = vec![source.clone()];
4480        for n in 1..=12 {
4481            runs.push(f.run(&format!("{n:04}"), false).await);
4482        }
4483        let dirty = runs[1].candidates[0].worktree.clone();
4484        std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4485        let ignored = runs[2].candidates[0].worktree.clone();
4486        std::fs::create_dir(ignored.join("scratch")).unwrap();
4487        std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4488        // A missing candidate array does not prove that an unrecorded worktree
4489        // is empty: discovery must inspect Git and the worktree itself.
4490        runs[1].candidates.clear();
4491        runs[1].save_under(&f.home).unwrap();
4492        let task = f.task(&runs);
4493        let mut total = 0;
4494        for _ in 0..4 {
4495            let mut report = Report::default();
4496            retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4497            assert!(report.retired <= 5);
4498            total += report.retired;
4499        }
4500        assert_eq!(total, 10);
4501        assert!(dirty.exists());
4502        assert!(ignored.exists());
4503        assert_eq!(
4504            git::rev_parse(&f.repo, &source.candidates[0].branch)
4505                .await
4506                .unwrap(),
4507            source_head
4508        );
4509        let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4510        assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4511        assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4512        let mut released = task.clone();
4513        released.release();
4514        assert_eq!(
4515            crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4516                RunState::load_under(id, &f.home)
4517            }),
4518            Some(source.id.clone())
4519        );
4520
4521        assert!(
4522            f.home
4523                .join("runs")
4524                .join(&retired.id)
4525                .join("run.json")
4526                .exists()
4527        );
4528        assert!(
4529            f.home
4530                .join("recovery-lineages")
4531                .join(format!("{}.json", task.id))
4532                .exists()
4533        );
4534    }
4535
4536    #[tokio::test]
4537    async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4538        let f = Fx::new().await;
4539        let a = f.run("dddd", true).await;
4540        let b = f.run("eeee", true).await;
4541        let empty = f.run("ffff", false).await;
4542        let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4543        assert!(
4544            crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4545            "unfinished queue history protects older source branches from ordinary folding"
4546        );
4547        let mut prior = a.clone();
4548        assert!(
4549            crate::graph::fold_run(&mut prior, true, &f.home)
4550                .await
4551                .unwrap_err()
4552                .to_string()
4553                .contains(&task.id)
4554        );
4555        let mut report = Report::default();
4556        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4557        assert_eq!(report.retired, 0);
4558        assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4559        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4560        Fx::target(&a.candidates[0].worktree);
4561        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4562        assert!(
4563            report.freed_bytes > 0,
4564            "source retention does not retain regenerable outputs"
4565        );
4566        assert!(a.candidates[0].worktree.exists());
4567        assert!(b.candidates[0].worktree.exists());
4568        assert!(empty.candidates[0].worktree.exists());
4569    }
4570
4571    #[tokio::test]
4572    async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4573        let f = Fx::new().await;
4574        let authority = f.run("p000", true).await;
4575        let question_run = f.run("p001", false).await;
4576        let mut checkpoint = f.run("p002", false).await;
4577        checkpoint.land_approval = Some(crate::run::LandApproval {
4578            question: "pending".into(),
4579            head: f.base.clone(),
4580        });
4581        checkpoint.save_under(&f.home).unwrap();
4582        let retained = f.run("p003", false).await;
4583        std::fs::write(
4584            f.home
4585                .join("runs")
4586                .join(&retained.id)
4587                .join("retain-recovery-sources"),
4588            "owner",
4589        )
4590        .unwrap();
4591        let mut partial = f.run("p004", false).await;
4592        let reviewer = partial.worktree_root().join("review-1");
4593        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4594            .await
4595            .unwrap();
4596        // Model a crash after the retirement fence but only one removal.
4597        partial.retired_to = Some(authority.id.clone());
4598        partial.save_under(&f.home).unwrap();
4599        assert!(
4600            git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4601                .await
4602                .unwrap()
4603        );
4604        let git_checkpoint = f.run("p005", false).await;
4605        let location = git::git(
4606            &git_checkpoint.candidates[0].worktree,
4607            &[
4608                "rev-parse",
4609                "--path-format=absolute",
4610                "--git-path",
4611                "sequencer",
4612            ],
4613        )
4614        .await
4615        .unwrap();
4616        std::fs::create_dir(location.trim()).unwrap();
4617        let mut preservation = f.run("p006", false).await;
4618        preservation
4619            .candidate_preservation
4620            .insert('A', "pending preservation".into());
4621        preservation.save_under(&f.home).unwrap();
4622        let task = f.task(&[
4623            preservation.clone(),
4624            git_checkpoint.clone(),
4625            authority.clone(),
4626            question_run.clone(),
4627            checkpoint.clone(),
4628            retained.clone(),
4629            partial.clone(),
4630        ]);
4631        let questions = crate::ask::Questions::at(f.home.join("questions"));
4632        let mut question = crate::ask::Question::new(
4633            question_run.id.clone(),
4634            "implement".into(),
4635            "fixture".into(),
4636            "owner decision".into(),
4637            String::new(),
4638            vec!["keep".into()],
4639        );
4640        questions.put(&mut question).unwrap();
4641        let mut report = Report::default();
4642        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4643        assert_eq!(report.retired, 1, "{report:?}");
4644        assert!(!reviewer.exists());
4645        for state in [
4646            &question_run,
4647            &checkpoint,
4648            &retained,
4649            &git_checkpoint,
4650            &preservation,
4651        ] {
4652            assert!(state.candidates[0].worktree.exists());
4653            assert!(
4654                RunState::load_under(&state.id, &f.home)
4655                    .unwrap()
4656                    .retired_to
4657                    .is_none()
4658            );
4659        }
4660        assert!(questions.get(&question.id).unwrap().status.open());
4661        assert_eq!(
4662            crate::queue::Queue::at(f.home.join("queue"))
4663                .get(&task.id)
4664                .unwrap()
4665                .status,
4666            task.status
4667        );
4668    }
4669
4670    #[tokio::test]
4671    async fn preservation_schema_migration_remains_distinct_from_retirement() {
4672        let f = Fx::new().await;
4673        let mut state = f.run("p007", false).await;
4674        state.schema = 15;
4675        state.save_under(&f.home).unwrap();
4676        let native = RunState::load_under(&state.id, &f.home).unwrap();
4677        assert_eq!(native.schema, crate::run::SCHEMA);
4678        assert!(native.candidate_preservation.is_empty());
4679        assert!(native.retired_to.is_none());
4680        state.schema = 14;
4681        state.save_under(&f.home).unwrap();
4682        let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4683        assert!(legacy.candidate_preservation.contains_key(&'A'));
4684        legacy.save_under(&f.home).unwrap();
4685        let restored = RunState::load_under(&state.id, &f.home).unwrap();
4686        assert_eq!(
4687            restored.candidate_preservation,
4688            legacy.candidate_preservation
4689        );
4690    }
4691
4692    #[tokio::test]
4693    async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4694        let f = Fx::new().await;
4695        let cache = f.repo.join("cache");
4696        let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4697        std::fs::create_dir_all(&cache).unwrap();
4698        assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4699        assert!(
4700            crate::cache::maintenance_prune(&f.home, &cache, 1)
4701                .unwrap()
4702                .is_none()
4703        );
4704        drop(guard);
4705        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4706    }
4707
4708    #[tokio::test]
4709    async fn launcher_death_does_not_release_live_tool_output_reservations() {
4710        let f = Fx::new().await;
4711        let run = f.run("abcd", false).await;
4712        let wt = &run.candidates[0].worktree;
4713        let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4714        helper
4715            .quiet()
4716            .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4717            .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4718            .env("MAGI_TEST_WORKTREE", wt)
4719            .stdin(std::process::Stdio::null())
4720            .stdout(std::process::Stdio::null())
4721            .stderr(std::process::Stdio::null())
4722            .kill_on_drop(true);
4723        assert!(
4724            tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4725                .await
4726                .unwrap()
4727                .unwrap()
4728                .success()
4729        );
4730        let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4731            .unwrap()
4732            .trim()
4733            .parse()
4734            .unwrap();
4735        struct KillTool(i32);
4736        impl Drop for KillTool {
4737            fn drop(&mut self) {
4738                // SAFETY: PID belongs to the bounded helper above.
4739                unsafe { libc::kill(self.0, libc::SIGKILL) };
4740            }
4741        }
4742        let tool = KillTool(pid);
4743        assert!(OutputLease::take(wt, true).is_err());
4744        assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4745        drop(tool);
4746        tokio::time::timeout(std::time::Duration::from_secs(5), async {
4747            loop {
4748                if OutputLease::take(wt, true).is_ok()
4749                    && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4750                {
4751                    break;
4752                }
4753                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4754            }
4755        })
4756        .await
4757        .unwrap();
4758    }
4759
4760    #[tokio::test]
4761    async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4762        let f = Fx::new().await;
4763        let run = f.run("abcd", false).await;
4764        let wt = &run.candidates[0].worktree;
4765        let cache = f.repo.join("cache");
4766        std::fs::create_dir(&cache).unwrap();
4767        for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4768            let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4769            let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4770            cmd.quiet()
4771                .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4772                .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4773                .stdin(std::process::Stdio::null())
4774                .stdout(std::process::Stdio::null())
4775                .stderr(std::process::Stdio::null())
4776                .kill_on_drop(true);
4777            if clear_owner {
4778                cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4779            } else {
4780                cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4781            }
4782            let gib = 1024 * 1024 * 1024;
4783            let mut leases = vec![
4784                OutputLease::borrow(wt).unwrap().unwrap(),
4785                OutputLease::cache(&f.home, &cache, false).unwrap(),
4786            ];
4787            leases.extend(
4788                OutputLease::admit_with(
4789                    &f.home,
4790                    std::slice::from_ref(wt),
4791                    8 * gib,
4792                    |_| Ok(9 * gib),
4793                )
4794                .unwrap(),
4795            );
4796            let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4797            assert!(
4798                tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4799                    .await
4800                    .unwrap()
4801                    .unwrap()
4802                    .success()
4803            );
4804            let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4805            struct KillTool(Option<i32>);
4806            impl Drop for KillTool {
4807                fn drop(&mut self) {
4808                    // SAFETY: PID from our helper; kill also bounds panic paths.
4809                    if let Some(pid) = self.0 {
4810                        unsafe { libc::kill(pid, libc::SIGKILL) };
4811                    }
4812                }
4813            }
4814            let mut tool = KillTool(Some(pid));
4815            // The CLI has already exited and cannot supply a parent-tree
4816            // relationship. Either the witness or inherited environment must
4817            // retain ownership; test each independently.
4818            assert!(OutputLease::take(wt, true).is_err());
4819            let finish = if cancel {
4820                drop(running);
4821                None
4822            } else {
4823                Some(tokio::spawn(async move { running.finish().await }))
4824            };
4825            if clear_owner {
4826                tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4827                assert!(crate::proc::pid_alive(pid as u32));
4828                assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4829                assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4830                assert!(
4831                    OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4832                        9 * gib
4833                    ))
4834                    .is_err(),
4835                    "orphan still owns its disk reservation"
4836                );
4837                assert!(
4838                    crate::cache::maintenance_prune(&f.home, &cache, 0)
4839                        .unwrap()
4840                        .is_none()
4841                );
4842                if let Some(finish) = &finish {
4843                    assert!(!finish.is_finished());
4844                }
4845                // Witness-only ownership waits for natural/external tool exit.
4846                unsafe { libc::kill(pid, libc::SIGKILL) };
4847            }
4848            if let Some(finish) = finish {
4849                tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4850                    .await
4851                    .unwrap()
4852                    .unwrap();
4853            }
4854            tokio::time::timeout(std::time::Duration::from_secs(5), async {
4855                loop {
4856                    if OutputLease::take(wt, true).is_ok()
4857                        && OutputLease::cache(&f.home, &cache, true).is_ok()
4858                        && OutputLease::admit_with(
4859                            &f.home,
4860                            std::slice::from_ref(wt),
4861                            8 * gib,
4862                            |_| Ok(9 * gib),
4863                        )
4864                        .is_ok()
4865                    {
4866                        // Token-owned orphans must be stopped by cleanup itself.
4867                        if !clear_owner {
4868                            assert!(!crate::proc::pid_alive(pid as u32));
4869                        }
4870                        break;
4871                    }
4872                    tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4873                }
4874            })
4875            .await
4876            .unwrap();
4877            tool.0 = None;
4878        }
4879    }
4880
4881    #[tokio::test]
4882    async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4883        let f = Fx::new().await;
4884        let run = f.run("abcd", false).await;
4885        let wt = &run.candidates[0].worktree;
4886        let cache = f.repo.join("cache");
4887        std::fs::create_dir(&cache).unwrap();
4888        let pid_file = f.repo.join("child-pid");
4889        let mut cmd = tokio::process::Command::new("sh");
4890        cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4891            .quiet()
4892            .env("CHILD_PID_FILE", &pid_file)
4893            .process_group(0)
4894            .kill_on_drop(true);
4895        let running = crate::seat_process::Running::spawn(
4896            cmd,
4897            None,
4898            vec![
4899                OutputLease::borrow(wt).unwrap().unwrap(),
4900                OutputLease::cache(&f.home, &cache, false).unwrap(),
4901            ],
4902        )
4903        .unwrap();
4904        tokio::time::timeout(std::time::Duration::from_secs(5), async {
4905            while !pid_file.exists() {
4906                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4907            }
4908        })
4909        .await
4910        .unwrap();
4911        assert!(OutputLease::take(wt, true).is_err());
4912        assert!(
4913            crate::cache::maintenance_prune(&f.home, &cache, 0)
4914                .unwrap()
4915                .is_none()
4916        );
4917        let pid: u32 = std::fs::read_to_string(pid_file)
4918            .unwrap()
4919            .trim()
4920            .parse()
4921            .unwrap();
4922        drop(running);
4923        tokio::time::timeout(std::time::Duration::from_secs(10), async {
4924            loop {
4925                if let Ok(guard) = OutputLease::take(wt, true) {
4926                    assert!(
4927                        !crate::proc::pid_alive(pid),
4928                        "output lock released before tool death"
4929                    );
4930                    drop(guard);
4931                    break;
4932                }
4933                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4934            }
4935        })
4936        .await
4937        .unwrap();
4938        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4939    }
4940
4941    #[tokio::test]
4942    async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
4943        let f = Fx::new().await;
4944        let reason = tokio::time::timeout(
4945            std::time::Duration::from_secs(2),
4946            emergency(vec![f.repo.clone()], u64::MAX),
4947        )
4948        .await
4949        .unwrap();
4950        assert!(reason.contains("disk pressure"));
4951        assert!(
4952            tokio::time::timeout(
4953                std::time::Duration::from_millis(10),
4954                emergency(vec![f.repo.clone()], 0)
4955            )
4956            .await
4957            .is_err()
4958        );
4959        assert_eq!(
4960            std::fs::read_to_string(f.repo.join("source")).unwrap(),
4961            "base\n"
4962        );
4963    }
4964
4965    #[tokio::test]
4966    async fn automatic_requeues_cannot_refund_failed_input_generation() {
4967        let f = Fx::new().await;
4968        let mut run = f.run("1234", false).await;
4969        run.prep_failure = Some("cherry-pick conflict".into());
4970        let mut task = f.task(std::slice::from_ref(&run));
4971        let inputs = crate::queue::recovery_inputs(&run.sources);
4972        for _ in 0..108 {
4973            task.recovery_failed(&run);
4974            task.requeue();
4975            assert_eq!(task.attempts, 0);
4976        }
4977        let bytes = serde_json::to_vec(&task).unwrap();
4978        let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
4979        assert!(
4980            restored
4981                .recovery_exhausted(&run.base_commit, &inputs, 2)
4982                .is_some()
4983        );
4984        assert!(
4985            restored
4986                .recovery_exhausted("different-base", &inputs, 2)
4987                .is_none()
4988        );
4989        let original_base = run.base_commit.clone();
4990        run.base_commit = "different-base".into();
4991        restored.recovery_failed(&run);
4992        assert!(
4993            restored
4994                .recovery_exhausted(&original_base, &inputs, 2)
4995                .is_some(),
4996            "returning to old inputs does not replenish their budget"
4997        );
4998        assert!(
4999            restored
5000                .recovery_exhausted(&run.base_commit, &inputs, 2)
5001                .is_none()
5002        );
5003        restored.release_operator("test-owner", None);
5004        assert!(
5005            restored
5006                .recovery_exhausted(&run.base_commit, &inputs, 2)
5007                .is_none()
5008        );
5009    }
5010    async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5011        let mut state = f.run(tag, useful).await;
5012        state.status = RunStatus::Superseded;
5013        state.save_under(&f.home).unwrap();
5014        state
5015    }
5016
5017    fn historical_review(state: &mut RunState, head: &str) {
5018        state.reviews.push(
5019            serde_json::from_value(serde_json::json!({
5020                "round": state.reviews.len() + 1, "head": head, "reviews": []
5021            }))
5022            .unwrap(),
5023        );
5024    }
5025
5026    async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5027        let path = state.worktree_root().join("review-1");
5028        git::worktree_add_detached(&f.repo, &path, head)
5029            .await
5030            .unwrap();
5031        path
5032    }
5033
5034    #[tokio::test]
5035    async fn source_eviction_separates_run_authority_from_read_seats() {
5036        for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5037            let f = Fx::new().await;
5038            let mut state = inactive_source(&f, "ef01", true).await;
5039            state.status = status;
5040            state.pr = Some(
5041                serde_json::from_value(serde_json::json!({
5042                    "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5043                    "checks":"green", "round":1, "rounds":1
5044                }))
5045                .unwrap(),
5046            );
5047            state.pr_head = Some(state.candidates[0].branch.clone());
5048            state.pr_merged_head = Some(f.base.clone());
5049            state.recovery = Some(
5050                serde_json::from_value(serde_json::json!({
5051                    "from_run": "20260101-000000-abcd", "generation": 1,
5052                    "mode": "full-review"
5053                }))
5054                .unwrap(),
5055            );
5056            state.pr_recovery = Some(
5057                serde_json::from_value(serde_json::json!({
5058                    "url": "https://example.invalid/pull/1", "branch": "public",
5059                    "head": f.base, "source_run": state.id,
5060                    "instruction": "keep implementation", "approved_at": Fx::now()
5061                }))
5062                .unwrap(),
5063            );
5064            state.save_under(&f.home).unwrap();
5065            let review = reviewer(&f, &state, &f.base).await;
5066            let mut other = inactive_source(&f, "ef02", true).await;
5067            other.pr_recovery = state.pr_recovery.clone();
5068            other.save_under(&f.home).unwrap();
5069            let mut task = f.task(&[state.clone(), other]);
5070            task.pr_recovery = state.pr_recovery.clone();
5071            crate::queue::Queue::at(f.home.join("queue"))
5072                .put(&mut task)
5073                .unwrap();
5074            let run_path = f.home.join("runs").join(&state.id).join("run.json");
5075            let before = std::fs::read(&run_path).unwrap();
5076            let task_before = serde_json::to_value(&task).unwrap();
5077            let mut report = Report::default();
5078            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5079            assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5080            assert!(!review.exists());
5081            assert!(state.candidates[0].worktree.exists());
5082            assert!(
5083                git::branch_exists(&f.repo, &state.candidates[0].branch)
5084                    .await
5085                    .unwrap()
5086            );
5087            assert_eq!(std::fs::read(&run_path).unwrap(), before);
5088            assert_eq!(
5089                serde_json::to_value(
5090                    crate::queue::Queue::at(f.home.join("queue"))
5091                        .get(&task.id)
5092                        .unwrap()
5093                )
5094                .unwrap(),
5095                task_before
5096            );
5097        }
5098    }
5099
5100    #[tokio::test]
5101    async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5102        for kind in ["own-path", "active-seat", "paused-session"] {
5103            let f = Fx::new().await;
5104            let mut state = inactive_source(&f, "ef03", true).await;
5105            let review = reviewer(&f, &state, &f.base).await;
5106            let expected = match kind {
5107                "own-path" => {
5108                    state.attachments.push(review.join("source"));
5109                    "run.attachments[0]"
5110                }
5111                "active-seat" => {
5112                    state.active.insert(
5113                        "review-1".into(),
5114                        serde_json::from_value(serde_json::json!({
5115                            "node":"review", "started_at":Fx::now(), "timeout_secs":60
5116                        }))
5117                        .unwrap(),
5118                    );
5119                    "continuation lease"
5120                }
5121                _ => {
5122                    state.status = RunStatus::Blocked;
5123                    state.parked = true;
5124                    let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5125                    seat.turns = 2;
5126                    seat.captured_session = Some("saved-session".into());
5127                    state.seats.insert("review-1".into(), seat);
5128                    "resume contract"
5129                }
5130            };
5131            state.save_under(&f.home).unwrap();
5132            let mut report = Report::default();
5133            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5134            assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5135            assert!(review.exists());
5136            assert!(
5137                report.retained.iter().any(|r| r.contains(expected)),
5138                "{kind}: {report:?}"
5139            );
5140            // The execution restoration boundary leaves protected existing
5141            // seats and their sessions untouched; prep need not recreate them.
5142            restore_execution_seats(&f.home, &state).await.unwrap();
5143            assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5144        }
5145    }
5146
5147    #[tokio::test]
5148    async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5149        for kind in ["waiting", "lease", "historical", "unknown"] {
5150            let f = Fx::new().await;
5151            let state = inactive_source(&f, "ef06", true).await;
5152            let review = reviewer(&f, &state, &f.base).await;
5153            let store = crate::ask::Questions::at(f.home.join("questions"));
5154            let mut question = crate::ask::Question::new(
5155                "20260101-000000-ffff".into(),
5156                "review".into(),
5157                "review-1".into(),
5158                "fixture".into(),
5159                "fixture".into(),
5160                Vec::new(),
5161            );
5162            question.cwd = Some(review.to_string_lossy().into_owned());
5163            if kind != "waiting" {
5164                question.status = crate::ask::QuestionStatus::Abandoned;
5165            }
5166            store.put(&mut question).unwrap();
5167            if kind == "lease" {
5168                store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5169            }
5170            if kind == "unknown" {
5171                let path = store.path_of(&question.id);
5172                let mut raw: serde_json::Value =
5173                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5174                raw["future_cwd_authority"] = serde_json::Value::Null;
5175                std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5176            }
5177            let mut report = Report::default();
5178            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5179            assert_eq!(
5180                report.evicted_copies,
5181                usize::from(kind == "historical"),
5182                "{kind}: {report:?}"
5183            );
5184            assert_eq!(review.exists(), kind != "historical");
5185            if kind == "waiting" || kind == "lease" {
5186                assert!(
5187                    report
5188                        .retained
5189                        .iter()
5190                        .any(|r| r.contains("question") && r.contains("cwd")),
5191                    "{report:?}"
5192                );
5193            }
5194        }
5195    }
5196
5197    #[tokio::test]
5198    async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5199        let f = Fx::new().await;
5200        let mut state = inactive_source(&f, "ef07", true).await;
5201        let review = reviewer(&f, &state, &f.base).await;
5202        let mut report = Report::default();
5203        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5204        assert_eq!(report.evicted_copies, 1, "{report:?}");
5205        state.config.worktree.setup = vec![crate::config::SetupStep {
5206            run: Some("printf preserved > source; exit 1".into()),
5207            ..Default::default()
5208        }];
5209        state.save_under(&f.home).unwrap();
5210        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5211        let marker = f
5212            .home
5213            .join("runs")
5214            .join(&state.id)
5215            .join("artifacts/restore-review-1.pending");
5216        assert!(marker.exists());
5217        assert_eq!(
5218            std::fs::read_to_string(review.join("source")).unwrap(),
5219            "preserved"
5220        );
5221        assert!(
5222            restore_execution_seats(&f.home, &state)
5223                .await
5224                .unwrap_err()
5225                .to_string()
5226                .contains("interrupted execution restoration")
5227        );
5228        let mut retry = Report::default();
5229        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5230        assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5231        assert!(
5232            retry
5233                .retained
5234                .iter()
5235                .any(|r| r.contains("restoration checkpoint")),
5236            "{retry:?}"
5237        );
5238        assert!(marker.exists());
5239        assert_eq!(
5240            std::fs::read_to_string(review.join("source")).unwrap(),
5241            "preserved"
5242        );
5243    }
5244
5245    #[tokio::test]
5246    async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5247        for kind in [
5248            "alternates",
5249            "promisor-config",
5250            "promisor-pack",
5251            "objects-symlink",
5252        ] {
5253            let f = Fx::new().await;
5254            let state = inactive_source(&f, "ef11", true).await;
5255            let review = reviewer(&f, &state, &f.base).await;
5256            let common = crate::repo_lock::common_dir(&f.repo)
5257                .await
5258                .unwrap()
5259                .unwrap();
5260            match kind {
5261                "alternates" => {
5262                    let borrowed = f._temp.path().join("borrowed-objects");
5263                    std::fs::create_dir(&borrowed).unwrap();
5264                    std::fs::write(
5265                        common.join("objects/info/alternates"),
5266                        format!("{}\n", borrowed.display()),
5267                    )
5268                    .unwrap();
5269                }
5270                "promisor-config" => {
5271                    git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5272                        .await
5273                        .unwrap();
5274                }
5275                "promisor-pack" => {
5276                    std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5277                }
5278                _ => {
5279                    let borrowed = f._temp.path().join("borrowed-objects");
5280                    std::fs::rename(common.join("objects"), &borrowed).unwrap();
5281                    std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5282                }
5283            }
5284            let mut report = Report::default();
5285            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5286            assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5287            assert!(review.exists());
5288            assert!(
5289                report.retained.iter().any(|r| r.contains("source anchor")),
5290                "{kind}: {report:?}"
5291            );
5292        }
5293    }
5294
5295    #[tokio::test]
5296    async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5297        let mut f = Fx::new().await;
5298        crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5299        f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5300        assert_eq!(
5301            git::git(&f.repo, &["rev-list", "--count", &f.base])
5302                .await
5303                .unwrap(),
5304            "5001"
5305        );
5306        let state = inactive_source(&f, "ef10", true).await;
5307        let review = reviewer(&f, &state, &f.base).await;
5308        std::fs::write(review.join("source"), "bounded review delta").unwrap();
5309        git::git(&review, &["add", "."]).await.unwrap();
5310        let tree = git::git(&review, &["write-tree"]).await.unwrap();
5311        let head = git::git(
5312            &review,
5313            &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5314        )
5315        .await
5316        .unwrap();
5317        git::git(&review, &["checkout", "--detach", &head])
5318            .await
5319            .unwrap();
5320        let run_file = f.home.join("runs").join(&state.id).join("run.json");
5321        let run_before = std::fs::read(&run_file).unwrap();
5322        let mut report = Report::default();
5323        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5324        assert_eq!(report.evicted_copies, 1, "{report:?}");
5325        assert!(!review.exists());
5326        assert!(state.candidates[0].worktree.exists());
5327        assert!(
5328            git::branch_exists(&f.repo, &state.candidates[0].branch)
5329                .await
5330                .unwrap()
5331        );
5332        assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5333        let archive = f
5334            .home
5335            .join("runs")
5336            .join(&state.id)
5337            .join("artifacts/source-evidence")
5338            .join(format!("{head}-{}", f.base));
5339        let manifest: serde_json::Value =
5340            serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5341        assert_eq!(manifest["anchor"], f.base);
5342        assert!(
5343            std::fs::metadata(archive.join("objects.pack"))
5344                .unwrap()
5345                .len()
5346                < 16 * 1024
5347        );
5348        let pin = format!("refs/magi/source-anchors/{}", f.base);
5349        assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5350        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5351            .await
5352            .unwrap();
5353        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5354        assert!(
5355            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5356                .await
5357                .unwrap()
5358                .ok()
5359        );
5360        let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5361            .await
5362            .unwrap();
5363        // Missing, moved, symbolic and symlink pins cannot authorize import.
5364        for kind in ["missing", "moved", "symbolic", "symlink"] {
5365            git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5366                .await
5367                .unwrap();
5368            match kind {
5369                "moved" => {
5370                    git::git(&f.repo, &["update-ref", &pin, &other])
5371                        .await
5372                        .unwrap();
5373                }
5374                "symbolic" => {
5375                    git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5376                        .await
5377                        .unwrap();
5378                }
5379                "symlink" => {
5380                    let common = crate::repo_lock::common_dir(&f.repo)
5381                        .await
5382                        .unwrap()
5383                        .unwrap();
5384                    let target = f._temp.path().join("operator-anchor");
5385                    std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5386                    let file = common.join(&pin);
5387                    std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5388                    std::os::unix::fs::symlink(target, file).unwrap();
5389                }
5390                _ => {}
5391            }
5392            assert!(
5393                restore_execution_seats(&f.home, &state).await.is_err(),
5394                "{kind}"
5395            );
5396            assert!(!review.exists());
5397            assert!(
5398                !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5399                    .await
5400                    .unwrap()
5401                    .ok()
5402            );
5403            if kind == "symlink" {
5404                let common = crate::repo_lock::common_dir(&f.repo)
5405                    .await
5406                    .unwrap()
5407                    .unwrap();
5408                std::fs::remove_file(common.join(&pin)).unwrap();
5409            } else {
5410                git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5411                    .await
5412                    .unwrap();
5413            }
5414            git::git(&f.repo, &["update-ref", &pin, &f.base])
5415                .await
5416                .unwrap();
5417        }
5418        // Fixture-only ref deletion proves the pin, rather than an incidental
5419        // main/candidate branch, keeps all shared ancestors through GC.
5420        assert!(
5421            git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5422                .await
5423                .unwrap()
5424        );
5425        git::git(
5426            &f.repo,
5427            &[
5428                "update-ref",
5429                "-d",
5430                &format!("refs/heads/{}", state.candidates[0].branch),
5431            ],
5432        )
5433        .await
5434        .unwrap();
5435        git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5436            .await
5437            .unwrap();
5438        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5439            .await
5440            .unwrap();
5441        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5442        assert_eq!(
5443            git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5444                .await
5445                .unwrap(),
5446            pin
5447        );
5448        assert_eq!(
5449            git::git(&f.repo, &["rev-list", "--count", &f.base])
5450                .await
5451                .unwrap(),
5452            "5001"
5453        );
5454        restore_execution_seats(&f.home, &state).await.unwrap();
5455        assert_eq!(
5456            git::git(&review, &["rev-list", "--count", "HEAD"])
5457                .await
5458                .unwrap(),
5459            "5002"
5460        );
5461        assert_eq!(
5462            git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5463                .await
5464                .unwrap(),
5465            "false"
5466        );
5467        git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5468        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5469        let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5470        assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5471        std::fs::remove_dir_all(&f.repo).unwrap();
5472        assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5473        let inspection = f._temp.path().join("shared-history-inspection");
5474        restore_seat(&f.home, &state.id, &head, &inspection)
5475            .await
5476            .unwrap();
5477        assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5478        assert_eq!(
5479            git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5480                .await
5481                .unwrap(),
5482            "true"
5483        );
5484        assert_eq!(
5485            git::git(&inspection, &["log", "--format=%H"])
5486                .await
5487                .unwrap(),
5488            format!("{head}\n{}", f.base)
5489        );
5490        assert!(!inspection.join(".git/objects/info/alternates").exists());
5491        git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5492        git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5493    }
5494
5495    #[tokio::test]
5496    async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5497        let f = Fx::new().await;
5498        let state = inactive_source(&f, "ef08", true).await;
5499        let review = reviewer(&f, &state, &f.base).await;
5500        let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5501        let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5502            .await
5503            .unwrap();
5504        let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
5505            .await
5506            .unwrap();
5507        git::git(&review, &["checkout", "--detach", &head])
5508            .await
5509            .unwrap();
5510        let archive =
5511            crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
5512        let mut report = Report::default();
5513        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5514        assert_eq!(report.evicted_copies, 0, "{report:?}");
5515        assert!(review.exists());
5516        assert!(
5517            report
5518                .retained
5519                .iter()
5520                .any(|r| r.contains("incomplete archived commit ancestry")),
5521            "{report:?}"
5522        );
5523        // Simulate a historical checkout already removed by the old writer.
5524        assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
5525        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5526            .await
5527            .unwrap();
5528        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5529        for oid in [&parent, &head] {
5530            assert!(
5531                !git::git_raw(&f.repo, &["cat-file", "-e", oid])
5532                    .await
5533                    .unwrap()
5534                    .ok()
5535            );
5536        }
5537        let patch = source_diff(&archive).await.unwrap();
5538        assert!(
5539            crate::fold_evidence::import_snapshot(&archive, &f.repo)
5540                .await
5541                .is_err()
5542        );
5543        assert!(
5544            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5545                .await
5546                .unwrap()
5547                .ok()
5548        );
5549        git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5550        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5551        let inspection = f._temp.path().join("legacy-tree-inspection");
5552        restore_source(&archive, &inspection).await.unwrap();
5553        assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
5554        assert!(
5555            !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
5556                .await
5557                .unwrap()
5558                .ok()
5559        );
5560        git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5561        git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5562        assert_eq!(source_diff(&archive).await.unwrap(), patch);
5563        assert!(state.candidates[0].worktree.exists());
5564    }
5565
5566    #[tokio::test]
5567    async fn source_eviction_retains_ancestry_above_expanded_budget() {
5568        let f = Fx::new().await;
5569        let state = inactive_source(&f, "ef09", true).await;
5570        let review = reviewer(&f, &state, &f.base).await;
5571        std::fs::File::create(review.join("ancestor-only"))
5572            .unwrap()
5573            .set_len(64 * 1024 * 1024 + 1)
5574            .unwrap();
5575        git::git(&review, &["add", "."]).await.unwrap();
5576        let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5577        let parent = git::git(
5578            &review,
5579            &["commit-tree", &parent_tree, "-m", "large ancestor"],
5580        )
5581        .await
5582        .unwrap();
5583        std::fs::remove_file(review.join("ancestor-only")).unwrap();
5584        git::git(&review, &["add", "."]).await.unwrap();
5585        let tree = git::git(&review, &["write-tree"]).await.unwrap();
5586        let head = git::git(
5587            &review,
5588            &[
5589                "commit-tree",
5590                &tree,
5591                "-p",
5592                &parent,
5593                "-m",
5594                "small final tree",
5595            ],
5596        )
5597        .await
5598        .unwrap();
5599        git::git(&review, &["checkout", "--detach", &head])
5600            .await
5601            .unwrap();
5602        let mut report = Report::default();
5603        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5604        assert_eq!(report.evicted_copies, 0, "{report:?}");
5605        assert!(
5606            report
5607                .retained
5608                .iter()
5609                .any(|r| r.contains("expanded object limit exceeded")),
5610            "{report:?}"
5611        );
5612        assert!(review.exists());
5613        assert!(state.candidates[0].worktree.exists());
5614    }
5615
5616    #[tokio::test]
5617    async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
5618        let f = Fx::new().await;
5619        let state = inactive_source(&f, "ef04", true).await;
5620        let review = reviewer(&f, &state, &f.base).await;
5621        std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
5622        git::git(&review, &["add", "."]).await.unwrap();
5623        let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5624        let parent = git::git(
5625            &review,
5626            &["commit-tree", &parent_tree, "-m", "unrelated parent"],
5627        )
5628        .await
5629        .unwrap();
5630        std::fs::remove_file(review.join("ancestor-only")).unwrap();
5631        std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
5632        git::git(&review, &["add", "."]).await.unwrap();
5633        let tree = git::git(&review, &["write-tree"]).await.unwrap();
5634        let head = git::git(
5635            &review,
5636            &[
5637                "commit-tree",
5638                &tree,
5639                "-p",
5640                &parent,
5641                "-m",
5642                "unrelated review checkpoint",
5643            ],
5644        )
5645        .await
5646        .unwrap();
5647        git::git(&review, &["checkout", "--detach", &head])
5648            .await
5649            .unwrap();
5650        let mut report = Report::default();
5651        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5652        assert_eq!(report.evicted_copies, 1, "{report:?}");
5653        let ledger = f
5654            .home
5655            .join("runs")
5656            .join(&state.id)
5657            .join("artifacts/eviction-review-1.json");
5658        let original = std::fs::read(&ledger).unwrap();
5659        let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
5660        assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
5661        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5662            .await
5663            .unwrap();
5664        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5665        assert!(
5666            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5667                .await
5668                .unwrap()
5669                .ok()
5670        );
5671        assert!(
5672            !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
5673                .await
5674                .unwrap()
5675                .ok()
5676        );
5677        assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5678        assert!(
5679            seat_heads(&f.home, &state.id)
5680                .await
5681                .unwrap()
5682                .contains_key(&head)
5683        );
5684        restore_execution_seats(&f.home, &state).await.unwrap();
5685        assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
5686        assert_eq!(
5687            git::git(&review, &["log", "--format=%H"]).await.unwrap(),
5688            format!("{head}\n{parent}")
5689        );
5690        git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5691        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5692        let inspection = f._temp.path().join("complete-history-inspection");
5693        restore_seat(&f.home, &state.id, &head, &inspection)
5694            .await
5695            .unwrap();
5696        assert_eq!(
5697            git::git(&inspection, &["log", "--format=%H"])
5698                .await
5699                .unwrap(),
5700            format!("{head}\n{parent}")
5701        );
5702        git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5703        git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5704
5705        assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
5706        assert!(
5707            !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
5708                .await
5709                .unwrap()
5710                .ok()
5711        );
5712        assert_eq!(
5713            std::fs::read_to_string(review.join("source")).unwrap(),
5714            "unrelated review checkpoint"
5715        );
5716        std::fs::write(review.join("operator-notes"), "keep").unwrap();
5717        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5718        assert_eq!(
5719            std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5720            "keep"
5721        );
5722        std::fs::remove_file(review.join("operator-notes")).unwrap();
5723        restore_execution_seats(&f.home, &state).await.unwrap();
5724        let mut second = Report::default();
5725        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
5726        assert_eq!(second.evicted_copies, 1, "{second:?}");
5727        assert_eq!(std::fs::read(&ledger).unwrap(), original);
5728        assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
5729        // A historical v1 fixture has no explicit base field. Neither schema
5730        // may reinterpret its immutable archive when the owning run advances.
5731        let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
5732        legacy["version"] = 1.into();
5733        legacy.as_object_mut().unwrap().remove("base");
5734        std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
5735        let mut advanced = state.clone();
5736        advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5737            .await
5738            .unwrap();
5739        advanced.save_under(&f.home).unwrap();
5740        assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5741        // Corruption blocks execution before a replacement checkout is made.
5742        let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
5743        let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
5744        let patch = std::fs::read(archive.join("change.patch")).unwrap();
5745        std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
5746        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5747        assert!(!review.exists());
5748        std::fs::write(archive.join("change.patch"), patch).unwrap();
5749        std::fs::create_dir(&review).unwrap();
5750        std::fs::write(review.join("operator-notes"), "keep").unwrap();
5751        assert!(restore_execution_seats(&f.home, &state).await.is_err());
5752        assert_eq!(
5753            std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5754            "keep"
5755        );
5756    }
5757
5758    #[tokio::test]
5759    async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
5760        use std::os::unix::fs::PermissionsExt;
5761        let mut f = Fx::new().await;
5762        std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
5763        git::git(&f.repo, &["add", "."]).await.unwrap();
5764        git::git(&f.repo, &["commit", "-m", "binary base"])
5765            .await
5766            .unwrap();
5767        f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5768        let mut state = inactive_source(&f, "ec01", true).await;
5769        let candidate = state.candidates[0].worktree.clone();
5770        let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5771        historical_review(&mut state, &earlier);
5772        let review = reviewer(&f, &state, &earlier).await;
5773        std::fs::remove_file(candidate.join("source")).unwrap();
5774        std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5775        std::fs::write(candidate.join("script"), "run\n").unwrap();
5776        std::fs::set_permissions(
5777            candidate.join("script"),
5778            std::fs::Permissions::from_mode(0o755),
5779        )
5780        .unwrap();
5781        git::git(&candidate, &["add", "."]).await.unwrap();
5782        git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5783            .await
5784            .unwrap();
5785        let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5786        let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5787        let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5788            .await
5789            .unwrap();
5790        historical_review(&mut state, &head);
5791        historical_review(&mut state, &head);
5792        state.status = RunStatus::Superseded;
5793        state.save_under(&f.home).unwrap();
5794        let task = f.task(std::slice::from_ref(&state));
5795        let before = serde_json::to_value(&state).unwrap();
5796        git::git(&review, &["reset", "--hard", &head])
5797            .await
5798            .unwrap();
5799        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5800        assert_eq!(report.evicted_copies, 1, "{report:?}");
5801        assert!(!review.exists());
5802        assert!(candidate.exists(), "selected implementation survives");
5803        assert!(report.source_unlinked_logical_bytes > 0);
5804        assert!(report.source_unlinked_allocated_bytes.is_some());
5805        assert!(report.source_evidence_added_allocated_bytes.is_some());
5806        assert_eq!(
5807            serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5808            before
5809        );
5810        assert_eq!(
5811            crate::queue::Queue::at(f.home.join("queue"))
5812                .get(&task.id)
5813                .unwrap()
5814                .runs,
5815            task.runs
5816        );
5817        let archive = f
5818            .home
5819            .join("runs")
5820            .join(&state.id)
5821            .join("artifacts/source-evidence")
5822            .join(format!("{head}-{}", f.base));
5823        let patch = source_diff(&archive).await.unwrap();
5824        assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5825        assert!(String::from_utf8_lossy(&patch).contains("100755"));
5826        assert!(
5827            git::worktree_remove_clean(&f.repo, &candidate)
5828                .await
5829                .unwrap()
5830        );
5831        git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5832            .await
5833            .unwrap();
5834        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5835            .await
5836            .unwrap();
5837        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5838        assert!(
5839            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5840                .await
5841                .unwrap()
5842                .ok()
5843        );
5844        // Even the remaining base repository is unavailable to the reader.
5845        std::fs::remove_dir_all(&f.repo).unwrap();
5846        assert_eq!(source_diff(&archive).await.unwrap(), patch);
5847        let restored = f._temp.path().join("restored");
5848        restore_source(&archive, &restored).await.unwrap();
5849        assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5850        assert_eq!(
5851            std::fs::read(restored.join("binary")).unwrap(),
5852            b"new\0binary\xfe"
5853        );
5854        assert!(!restored.join("source").exists());
5855        assert_ne!(
5856            std::fs::metadata(restored.join("script"))
5857                .unwrap()
5858                .permissions()
5859                .mode()
5860                & 0o111,
5861            0
5862        );
5863        assert!(restore_source(&archive, &restored).await.is_err());
5864        // Reconstruct from the archived base plus the displayed binary patch,
5865        // rather than relying only on checking out the archived final tree.
5866        git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5867            .await
5868            .unwrap();
5869        let patch_file = f._temp.path().join("saved.patch");
5870        std::fs::write(&patch_file, &patch).unwrap();
5871        git::git(
5872            &restored,
5873            &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5874        )
5875        .await
5876        .unwrap();
5877        assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5878        assert_eq!(
5879            std::fs::read(restored.join("binary")).unwrap(),
5880            b"new\0binary\xfe"
5881        );
5882    }
5883
5884    #[tokio::test]
5885    async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5886        let f = Fx::new().await;
5887        let state = inactive_source(&f, "ec10", true).await;
5888        let review = reviewer(&f, &state, &f.base).await;
5889        let mut successor = inactive_source(&f, "ec11", true).await;
5890        successor.retired_to = Some(state.id.clone());
5891        successor.save_under(&f.home).unwrap();
5892        f.task(&[state.clone(), successor]);
5893
5894        let mut tips = Vec::new();
5895        for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5896            std::fs::write(review.join("source"), contents).unwrap();
5897            git::git(&review, &["add", "."]).await.unwrap();
5898            let tree = git::git(&review, &["write-tree"]).await.unwrap();
5899            let tip = git::git(
5900                &review,
5901                &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5902            )
5903            .await
5904            .unwrap();
5905            git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5906            tips.push(tip);
5907            git::git(&review, &["reset", "--hard", &f.base])
5908                .await
5909                .unwrap();
5910        }
5911        // Keep one tip exclusively in ORIG_HEAD, the other in the HEAD log.
5912        git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5913            .await
5914            .unwrap();
5915        git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5916            .await
5917            .unwrap();
5918        let mut report = Report::default();
5919        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5920        assert_eq!(report.evicted_copies, 1, "{report:?}");
5921        assert!(!review.exists());
5922        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5923            .await
5924            .unwrap();
5925        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5926        for tip in &tips {
5927            assert!(
5928                !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5929                    .await
5930                    .unwrap()
5931                    .ok()
5932            );
5933        }
5934        // The run-oriented interface also works without the source object store.
5935        std::fs::remove_dir_all(&f.repo).unwrap();
5936        let shown = seat_evidence(&f.home, &state.id).await.unwrap();
5937        for (i, tip) in tips.iter().enumerate() {
5938            assert!(
5939                shown
5940                    .iter()
5941                    .any(|row| row.contains(tip) && row.contains("patch_bytes="))
5942            );
5943            let destination = f._temp.path().join(format!("restore-history-{i}"));
5944            restore_seat(&f.home, &state.id, tip, &destination)
5945                .await
5946                .unwrap();
5947            assert_eq!(
5948                std::fs::read_to_string(destination.join("source")).unwrap(),
5949                ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
5950            );
5951        }
5952        assert!(
5953            restore_seat(
5954                &f.home,
5955                &state.id,
5956                "unknown",
5957                &f._temp.path().join("missing")
5958            )
5959            .await
5960            .is_err()
5961        );
5962    }
5963
5964    #[tokio::test]
5965    async fn source_eviction_revalidates_archive_immediately_before_unlink() {
5966        fn corrupt(path: &Path) -> Result<()> {
5967            let root = path.parent().unwrap();
5968            // Locate the fixture's durable artifact store through its run record
5969            // path supplied below by a test-only hook in the artifact tree.
5970            let marker = root.join("archive-to-corrupt");
5971            let archive = std::fs::read_to_string(&marker)?;
5972            std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
5973            Ok(())
5974        }
5975        let f = Fx::new().await;
5976        let state = inactive_source(&f, "ec12", true).await;
5977        let review = reviewer(&f, &state, &f.base).await;
5978        let archive = f
5979            .home
5980            .join("runs")
5981            .join(&state.id)
5982            .join("artifacts/source-evidence")
5983            .join(format!("{}-{}", f.base, f.base));
5984        std::fs::write(
5985            state.worktree_root().join("archive-to-corrupt"),
5986            archive.to_str().unwrap(),
5987        )
5988        .unwrap();
5989        let mut report = Report::default();
5990        copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
5991        assert_eq!(report.evicted_copies, 0, "{report:?}");
5992        assert!(review.exists());
5993        assert!(
5994            report
5995                .retained
5996                .iter()
5997                .any(|r| r.contains("corrupt snapshot")),
5998            "{report:?}"
5999        );
6000    }
6001
6002    #[tokio::test]
6003    async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6004        for kind in [
6005            "dirty",
6006            "hidden",
6007            "ignored",
6008            "empty-directory",
6009            "untracked",
6010            "borrowed",
6011            "borrowed-symlink",
6012            "lease",
6013            "owner",
6014            "driver",
6015            "task-pin",
6016            "task-replacement",
6017            "malformed-orig-head",
6018            "grace",
6019            "resumable",
6020            "archive-symlink",
6021            "root-symlink",
6022            "checkpoint",
6023            "symlink",
6024            "retained",
6025            "unreadable",
6026            "unknown-field",
6027            "unknown-task-field",
6028        ] {
6029            let f = Fx::new().await;
6030            let mut state = inactive_source(&f, "ec02", true).await;
6031            let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6032                .await
6033                .unwrap();
6034            historical_review(&mut state, &head);
6035            let review = reviewer(&f, &state, &head).await;
6036            let mut output = None;
6037            let mut owner = None;
6038            let mut driver = None;
6039            let mut disk = Fx::disk();
6040            match kind {
6041                "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6042                "hidden" => {
6043                    git::git(&review, &["update-index", "--assume-unchanged", "source"])
6044                        .await
6045                        .unwrap();
6046                    std::fs::write(review.join("source"), "notes").unwrap();
6047                }
6048                "ignored" => {
6049                    std::fs::create_dir(review.join("scratch")).unwrap();
6050                    std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6051                }
6052                "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6053                "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6054                "borrowed" => {
6055                    let mut other = f.run("ec03", true).await;
6056                    other.candidates[0].worktree = review.clone();
6057                    other.save_under(&f.home).unwrap();
6058                }
6059                "borrowed-symlink" => {
6060                    let alias = f._temp.path().join("borrowed-alias");
6061                    std::os::unix::fs::symlink(&review, &alias).unwrap();
6062                    let mut other = f.run("ec03", true).await;
6063                    other.candidates[0].worktree = alias;
6064                    other.save_under(&f.home).unwrap();
6065                }
6066                "lease" => output = OutputLease::borrow(&review).unwrap(),
6067                "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6068                "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6069                "task-pin" => {
6070                    let mut task = f.task(std::slice::from_ref(&state));
6071                    task.resume_override = Some(crate::queue::OperatorResume {
6072                        question_id: "fixture-pin".into(),
6073                        at: Fx::now(),
6074                        conductor_rehold: None,
6075                        forced: true,
6076                        pinned_run: Some(state.id.clone()),
6077                    });
6078                    crate::queue::Queue::at(f.home.join("queue"))
6079                        .put(&mut task)
6080                        .unwrap();
6081                }
6082                "task-replacement" => {
6083                    let mut task = f.task(std::slice::from_ref(&state));
6084                    task.replacement = Some(crate::merge_decision::Replacement {
6085                        question: "fixture-replacement".into(),
6086                        run: state.id.clone(),
6087                        pr: "https://example.invalid/pull/1".into(),
6088                        agent: "worker".into(),
6089                        context: String::new(),
6090                    });
6091                    crate::queue::Queue::at(f.home.join("queue"))
6092                        .put(&mut task)
6093                        .unwrap();
6094                }
6095                "malformed-orig-head" => {
6096                    let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6097                        .await
6098                        .unwrap();
6099                    std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6100                        .unwrap();
6101                }
6102                "grace" => disk.fold_grace_secs = 3600,
6103                "resumable" => state.status = RunStatus::Blocked,
6104                "archive-symlink" => {
6105                    let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6106                    std::fs::create_dir_all(&artifacts).unwrap();
6107                    let outside = f._temp.path().join("outside");
6108                    std::fs::create_dir(&outside).unwrap();
6109                    std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6110                        .unwrap();
6111                }
6112                "root-symlink" => {
6113                    let root = state.worktree_root();
6114                    let outside = f._temp.path().join("outside-root");
6115                    std::fs::rename(&root, &outside).unwrap();
6116                    std::os::unix::fs::symlink(&outside, &root).unwrap();
6117                }
6118                "checkpoint" => {
6119                    let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6120                        .await
6121                        .unwrap();
6122                    std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6123                }
6124                "symlink" => {
6125                    std::fs::remove_file(review.join("source")).unwrap();
6126                    std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6127                }
6128                "retained" => std::fs::write(
6129                    f.home
6130                        .join("runs")
6131                        .join(&state.id)
6132                        .join("retain-recovery-sources"),
6133                    "owner",
6134                )
6135                .unwrap(),
6136                "unreadable" => {
6137                    std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6138                }
6139                "unknown-field" | "unknown-task-field" => {}
6140                _ => unreachable!(),
6141            }
6142            state.save_under(&f.home).unwrap();
6143            if kind == "unknown-field" {
6144                let path = f.home.join("runs").join(&state.id).join("run.json");
6145                let mut raw: serde_json::Value =
6146                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6147                raw["future_source_owner"] = serde_json::json!({"path": review});
6148                std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6149            }
6150            if kind == "unknown-task-field" {
6151                let task = f.task(std::slice::from_ref(&state));
6152                let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6153                let mut raw: serde_json::Value =
6154                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6155                raw["future_source_owner"] = serde_json::json!({"path": review});
6156                std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6157            }
6158            let mut report = Report::default();
6159            copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6160            assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6161            assert!(review.exists(), "{kind}: {report:?}");
6162            assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6163            drop(output);
6164            drop(owner);
6165            drop(driver);
6166        }
6167    }
6168
6169    #[tokio::test]
6170    async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6171        let f = Fx::new().await;
6172        let state = inactive_source(&f, "ec04", true).await;
6173        let review = reviewer(&f, &state, &f.base).await;
6174        let mut report = Report::default();
6175        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6176        assert_eq!(report.evicted_copies, 1, "{report:?}");
6177        let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6178        let ledger = artifacts.join("eviction-review-1.json");
6179        let mut record: serde_json::Value =
6180            serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6181        record["phase"] = "pending".into();
6182        std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6183        let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6184        let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6185        std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6186        let mut blocked = Report::default();
6187        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6188        assert!(
6189            blocked
6190                .retained
6191                .iter()
6192                .any(|r| r.contains("corrupt snapshot")),
6193            "{blocked:?}"
6194        );
6195        assert_eq!(
6196            serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6197                ["phase"],
6198            "pending"
6199        );
6200        assert!(
6201            restore_source(&archive, &f._temp.path().join("bad-restore"))
6202                .await
6203                .is_err()
6204        );
6205        std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6206        let mut retried = Report::default();
6207        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6208        assert_eq!(
6209            serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6210                ["phase"],
6211            "evicted"
6212        );
6213        assert_eq!(retried.evicted_copies, 0);
6214        assert!(!review.exists());
6215        // A stopped archive publisher can resume its immutable staging files.
6216        let pending = archive.with_extension("pending");
6217        std::fs::rename(&archive, &pending).unwrap();
6218        assert_eq!(
6219            crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6220                .await
6221                .unwrap(),
6222            archive
6223        );
6224        assert!(!pending.exists());
6225    }
6226
6227    #[tokio::test]
6228    async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6229        let f = Fx::new().await;
6230        let mut empty = inactive_source(&f, "ec05", false).await;
6231        historical_review(&mut empty, &f.base);
6232        empty.status = RunStatus::Superseded;
6233        empty.save_under(&f.home).unwrap();
6234        let source = inactive_source(&f, "ec06", true).await;
6235        let task = f.task(&[empty.clone(), source.clone()]);
6236        let before = serde_json::to_value(&empty).unwrap();
6237        let mut report = Report::default();
6238        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6239        assert_eq!(report.evicted_copies, 1, "{report:?}");
6240        assert!(!empty.candidates[0].worktree.exists());
6241        assert!(source.candidates[0].worktree.exists());
6242        assert!(
6243            git::branch_exists(&f.repo, &empty.candidates[0].branch)
6244                .await
6245                .unwrap()
6246        );
6247        assert_eq!(
6248            serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6249            before
6250        );
6251        let mut retry = Report::default();
6252        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6253        assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6254        assert!(!empty.worktree_root().exists());
6255        assert_eq!(
6256            crate::queue::Queue::at(f.home.join("queue"))
6257                .get(&task.id)
6258                .unwrap()
6259                .runs,
6260            task.runs
6261        );
6262    }
6263    #[tokio::test]
6264    async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6265        fn interrupt(_: &Path) -> Result<()> {
6266            anyhow::bail!("fixture interruption before unlink")
6267        }
6268        fn vanish(path: &Path) -> Result<()> {
6269            std::fs::remove_dir_all(path)?;
6270            anyhow::bail!("fixture interruption after checkout unlink")
6271        }
6272        for stage in [
6273            "intent",
6274            "missing-registration",
6275            "changed-admin",
6276            "missing-archive",
6277            "wrong-identity",
6278        ] {
6279            let f = Fx::new().await;
6280            let state = inactive_source(&f, "ec07", true).await;
6281            let review = reviewer(&f, &state, &f.base).await;
6282            let admin = PathBuf::from(
6283                git::git(&review, &["rev-parse", "--absolute-git-dir"])
6284                    .await
6285                    .unwrap(),
6286            );
6287            let mut interrupted = Report::default();
6288            copies::evict_with(
6289                &f.home,
6290                &Fx::disk(),
6291                Fx::now(),
6292                &mut interrupted,
6293                if stage == "missing-registration" || stage == "changed-admin" {
6294                    vanish
6295                } else {
6296                    interrupt
6297                },
6298            )
6299            .await;
6300            assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6301            let ledger = f
6302                .home
6303                .join("runs")
6304                .join(&state.id)
6305                .join("artifacts/eviction-review-1.json");
6306            let record: serde_json::Value =
6307                serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6308            assert_eq!(record["phase"], "pending");
6309            if stage == "changed-admin" {
6310                std::fs::write(admin.join("owner-note"), "keep").unwrap();
6311            }
6312            if stage == "missing-archive" {
6313                std::fs::remove_file(
6314                    Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6315                )
6316                .unwrap();
6317            }
6318            if stage == "wrong-identity" {
6319                let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6320                    .await
6321                    .unwrap();
6322                let other =
6323                    crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6324                        .await
6325                        .unwrap();
6326                let original = Path::new(record["evidence"].as_str().unwrap());
6327                for name in [
6328                    "manifest.json",
6329                    "objects.pack",
6330                    "change.patch",
6331                    "change.index",
6332                ] {
6333                    std::fs::copy(other.join(name), original.join(name)).unwrap();
6334                }
6335            }
6336            let mut retried = Report::default();
6337            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6338            let phase =
6339                serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6340                    .unwrap()["phase"]
6341                    .clone();
6342            if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6343                assert_eq!(phase, "pending", "{stage}: {retried:?}");
6344                assert!(!retried.retained.is_empty());
6345                if stage == "changed-admin" {
6346                    assert!(admin.join("owner-note").exists());
6347                } else {
6348                    assert!(review.exists());
6349                }
6350            } else {
6351                assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6352                assert!(!review.exists());
6353                assert!(!admin.exists());
6354            }
6355        }
6356    }
6357    #[tokio::test]
6358    async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
6359        let f = Fx::new().await;
6360        let mut runs = Vec::new();
6361        let mut paths = Vec::new();
6362        for tag in ["ec10", "ec11", "ec12", "ec13"] {
6363            let state = inactive_source(&f, tag, true).await;
6364            for name in ["judge-1", "advisor-1"] {
6365                let path = state.worktree_root().join(name);
6366                git::worktree_add_detached(&f.repo, &path, &f.base)
6367                    .await
6368                    .unwrap();
6369                paths.push(path);
6370            }
6371            runs.push(state);
6372        }
6373        let task = f.task(&runs);
6374        let mut report = Report::default();
6375        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6376        assert_eq!(report.evicted_copies, 5, "{report:?}");
6377        let mut next = Report::default();
6378        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6379        assert_eq!(next.evicted_copies, 3, "{next:?}");
6380        assert!(paths.iter().all(|p| !p.exists()));
6381        assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
6382        assert_eq!(
6383            crate::queue::Queue::at(f.home.join("queue"))
6384                .get(&task.id)
6385                .unwrap()
6386                .runs,
6387            task.runs
6388        );
6389        let limited = Fx::new().await;
6390        let state = inactive_source(&limited, "ec15", true).await;
6391        let copy = reviewer(&limited, &state, &limited.base).await;
6392        let evidence = limited
6393            .home
6394            .join("runs")
6395            .join(&state.id)
6396            .join("artifacts/source-evidence");
6397        std::fs::create_dir_all(&evidence).unwrap();
6398        let retained = evidence.join("reserved-evidence");
6399        std::fs::File::create(&retained)
6400            .unwrap()
6401            .set_len(64 * 1024 * 1024)
6402            .unwrap();
6403        let mut blocked = Report::default();
6404        copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6405        assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
6406        assert!(copy.exists());
6407        assert_eq!(
6408            std::fs::metadata(&retained).unwrap().len(),
6409            64 * 1024 * 1024
6410        );
6411        assert!(
6412            blocked
6413                .retained
6414                .iter()
6415                .any(|r| r.contains("per-run storage limit")),
6416            "{blocked:?}"
6417        );
6418    }
6419    #[tokio::test]
6420    async fn source_eviction_migrates_historical_records_without_rewriting_them() {
6421        let f = Fx::new().await;
6422        let mut state = inactive_source(&f, "ec14", true).await;
6423        historical_review(&mut state, &f.base);
6424        historical_review(&mut state, &f.base);
6425        state.status = RunStatus::Superseded;
6426        state.save_under(&f.home).unwrap();
6427        let review = reviewer(&f, &state, &f.base).await;
6428        let task = f.task(std::slice::from_ref(&state));
6429        let path = f.home.join("runs").join(&state.id).join("run.json");
6430        let mut raw: serde_json::Value =
6431            serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6432        raw["schema"] = 10.into();
6433        raw.as_object_mut().unwrap().remove("output_activity_at");
6434        // Historical writers persisted null/empty defaults that current serde
6435        // omits. These are known fields, not unknown recovery authorities.
6436        raw["resource_wait"] = serde_json::Value::Null;
6437        raw["resource_block"] = serde_json::Value::Null;
6438        raw["driver_claimed"] = serde_json::Value::Null;
6439        raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6440        // Actual legacy spelling accepted by Graph's serde alias.
6441        let graph = raw["config"]["graph"].as_object_mut().unwrap();
6442        let count = graph.remove("implementers").unwrap();
6443        graph.insert("candidates".into(), count);
6444        let queue = crate::queue::Queue::at(f.home.join("queue"));
6445        let task_path = queue.path_of(&task.id);
6446        let mut task_raw: serde_json::Value =
6447            serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
6448        task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6449        let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
6450        std::fs::write(&task_path, &task_historical).unwrap();
6451        let historical = serde_json::to_vec_pretty(&raw).unwrap();
6452        std::fs::write(&path, &historical).unwrap();
6453        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6454        assert_eq!(report.evicted_copies, 1, "{report:?}");
6455        assert!(!review.exists());
6456        assert_eq!(std::fs::read(&path).unwrap(), historical);
6457        assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
6458        assert!(state.candidates[0].worktree.exists());
6459        assert_eq!(
6460            crate::queue::Queue::at(f.home.join("queue"))
6461                .get(&task.id)
6462                .unwrap()
6463                .runs,
6464            task.runs
6465        );
6466    }
6467    #[tokio::test]
6468    async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
6469        let f = Fx::new().await;
6470        let state = inactive_source(&f, "ed01", true).await;
6471        let review = reviewer(&f, &state, &f.base).await;
6472        std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
6473        git::git(&review, &["add", "."]).await.unwrap();
6474        let tree = git::git(&review, &["write-tree"]).await.unwrap();
6475        let tip = git::git(
6476            &review,
6477            &[
6478                "commit-tree",
6479                &tree,
6480                "-p",
6481                &f.base,
6482                "-m",
6483                "temporary detached source",
6484            ],
6485        )
6486        .await
6487        .unwrap();
6488        git::git(&review, &["checkout", "--detach", &tip])
6489            .await
6490            .unwrap();
6491        git::git(&review, &["checkout", "--detach", &f.base])
6492            .await
6493            .unwrap();
6494        git::git(&review, &["reflog", "delete", "HEAD@{1}"])
6495            .await
6496            .unwrap();
6497        git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
6498            .await
6499            .unwrap();
6500        let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
6501            .await
6502            .unwrap();
6503        assert!(!shown.lines().any(|oid| oid == tip));
6504        let admin = PathBuf::from(
6505            git::git(&review, &["rev-parse", "--absolute-git-dir"])
6506                .await
6507                .unwrap(),
6508        );
6509        let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
6510        assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
6511        fn interrupt(_: &Path) -> Result<()> {
6512            anyhow::bail!("fixture interruption before unlink")
6513        }
6514        let mut pending = Report::default();
6515        copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
6516        assert!(review.exists());
6517        let ledger = f
6518            .home
6519            .join("runs")
6520            .join(&state.id)
6521            .join("artifacts/eviction-review-1.json");
6522        let complete = std::fs::read(&ledger).unwrap();
6523        let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
6524        incomplete["history"]
6525            .as_object_mut()
6526            .unwrap()
6527            .remove(&tip)
6528            .unwrap();
6529        std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
6530        let mut blocked = Report::default();
6531        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6532        assert!(review.exists());
6533        assert!(
6534            blocked
6535                .retained
6536                .iter()
6537                .any(|reason| reason.contains("incomplete reflog checkpoint")),
6538            "{blocked:?}"
6539        );
6540        std::fs::write(&ledger, complete).unwrap();
6541        let mut report = Report::default();
6542        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6543        assert_eq!(report.evicted_copies, 1, "{report:?}");
6544        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6545            .await
6546            .unwrap();
6547        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6548        assert!(
6549            !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
6550                .await
6551                .unwrap()
6552                .ok()
6553        );
6554        std::fs::remove_dir_all(&f.repo).unwrap();
6555        let destination = f._temp.path().join("restore-old-only");
6556        restore_seat(&f.home, &state.id, &tip, &destination)
6557            .await
6558            .unwrap();
6559        assert_eq!(
6560            std::fs::read(destination.join("old-only")).unwrap(),
6561            b"old-only\0binary\xff"
6562        );
6563    }
6564
6565    #[tokio::test]
6566    async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
6567        fn interrupt(_: &Path) -> Result<()> {
6568            anyhow::bail!("fixture interruption before unlink")
6569        }
6570        fn vanish(path: &Path) -> Result<()> {
6571            std::fs::remove_dir_all(path)?;
6572            anyhow::bail!("fixture interruption before unregistration")
6573        }
6574        for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
6575            let f = Fx::new().await;
6576            let mut state = inactive_source(&f, "ed02", true).await;
6577            state.driver_exited = false;
6578            state.driver_claimed = Some(true);
6579            state.driver_pid = Some(std::process::id());
6580            state.driver_started_at = crate::proc::process_started_at(std::process::id());
6581            state.begin_pass();
6582            state.status = RunStatus::Superseded;
6583            state.save_under(&f.home).unwrap();
6584            let review = reviewer(&f, &state, &f.base).await;
6585            // The same matching identity written by DriverGuard::record, left
6586            // behind when a process exits without its cleanup guard running.
6587            let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
6588                "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
6589            std::fs::write(
6590                f.home.join("runs").join(&state.id).join("driver.json"),
6591                serde_json::to_vec(&identity).unwrap(),
6592            )
6593            .unwrap();
6594            let live = if stage == "live-driver" {
6595                Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
6596            } else {
6597                None
6598            };
6599            assert_eq!(
6600                state.liveness_under(&f.home, false),
6601                if live.is_some() {
6602                    Liveness::Live
6603                } else {
6604                    Liveness::Dead
6605                }
6606            );
6607            let mut report = Report::default();
6608            if stage == "intent" || stage == "missing-registration" {
6609                copies::evict_with(
6610                    &f.home,
6611                    &Fx::disk(),
6612                    Fx::now(),
6613                    &mut report,
6614                    if stage == "intent" { interrupt } else { vanish },
6615                )
6616                .await;
6617                assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
6618                let mut retry = Report::default();
6619                copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6620                assert!(!review.exists(), "{stage}: {retry:?}");
6621                let ledger = f
6622                    .home
6623                    .join("runs")
6624                    .join(&state.id)
6625                    .join("artifacts/eviction-review-1.json");
6626                let receipt: serde_json::Value =
6627                    serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
6628                assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
6629            } else {
6630                copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6631                assert_eq!(
6632                    report.evicted_copies,
6633                    usize::from(live.is_none()),
6634                    "{stage}: {report:?}"
6635                );
6636                assert_eq!(review.exists(), live.is_some());
6637            }
6638            drop(live);
6639            assert!(
6640                !RunState::load_under(&state.id, &f.home)
6641                    .unwrap()
6642                    .driver_exited
6643            );
6644        }
6645    }
6646}