Skip to main content

magi/
retention.rs

1//! Bounded reclamation of retained attempts and Cargo outputs. Source recovery
2//! and build eviction have separate authority: evicting a target never retires a run.
3mod copies;
4pub mod reconcile;
5
6/// Read the binary diff after verifying its independent source archive.
7pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
8    crate::fold_evidence::validate_snapshot(evidence).await?;
9    Ok(std::fs::read(evidence.join("change.patch"))?)
10}
11
12/// Restore immutable checkout evidence into a new inspection repository.
13pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
14    crate::fold_evidence::restore_snapshot(evidence, destination).await
15}
16
17/// List archived checkout tips for a run, validating their durable evidence.
18pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
19    copies::seat_evidence(home, id).await
20}
21
22/// Restore a recorded checkout tip without changing run or recovery authority.
23pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
24    copies::restore_seat(home, id, head, destination).await
25}
26
27use crate::{
28    git,
29    proc::Quiet as _,
30    run::{Liveness, RunState},
31};
32use anyhow::{Context, Result, ensure};
33use jiff::Timestamp;
34use serde::{Deserialize, Serialize};
35use std::{
36    fs::{File, OpenOptions},
37    path::{Path, PathBuf},
38};
39
40/// An OS lease on a worktree's outputs, independent of repository coordination.
41/// Kept by the process cleanup owner until descendants terminate, including
42/// reparented tools retaining the inherited liveness witness.
43#[derive(Debug)]
44pub(crate) struct OutputLease {
45    _file: File,
46    _activity: bool,
47}
48
49impl OutputLease {
50    #[cfg(unix)]
51    pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
52        use std::os::fd::AsRawFd;
53        self._file.as_raw_fd()
54    }
55
56    pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
57        if !worktree.join(".git").is_file() {
58            return Ok(None);
59        }
60        Self::take(worktree, false).map(Some)
61    }
62    pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
63        use std::hash::{Hash, Hasher};
64        let mut hash = std::collections::hash_map::DefaultHasher::new();
65        let mut normalized = cache.to_owned();
66        let mut missing = Vec::new();
67        while !normalized.exists() {
68            let Some(name) = normalized.file_name() else {
69                break;
70            };
71            missing.push(name.to_owned());
72            if !normalized.pop() {
73                break;
74            }
75        }
76        normalized = normalized.canonicalize().unwrap_or(normalized);
77        for part in missing.into_iter().rev() {
78            normalized.push(part);
79        }
80        normalized.hash(&mut hash);
81        let dir = home.join("cache-leases");
82        std::fs::create_dir_all(&dir)?;
83        Self::lock(
84            &dir.join(format!("outputs-{:x}.lock", hash.finish())),
85            exclusive,
86        )
87    }
88    /// Reserve headroom once per volume while admitting a managed writer.
89    /// These are estimates, not filesystem quotas. The ordinary emergency
90    /// monitor remains necessary for writes exceeding the reservation.
91    #[cfg(unix)]
92    pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
93        Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
94    }
95
96    #[cfg(unix)]
97    pub(crate) fn admit_with(
98        home: &Path,
99        paths: &[PathBuf],
100        min_free: u64,
101        free_bytes: impl Fn(&Path) -> Result<u64>,
102    ) -> Result<Vec<Self>> {
103        use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
104        if min_free == 0 {
105            return Ok(Vec::new());
106        }
107        const RESERVE: u64 = 1024 * 1024 * 1024;
108        std::fs::create_dir_all(home)?;
109        let dir = home.canonicalize()?.join("disk-admission");
110        std::fs::create_dir_all(&dir)?;
111        ensure!(
112            plain_dir(&dir) && dir.canonicalize()? == dir,
113            "disk admission directory is not canonical"
114        );
115        let mut devices = std::collections::BTreeSet::new();
116        let mut reservations = Vec::new();
117        for path in paths {
118            let probe = path
119                .ancestors()
120                .find(|p| p.exists())
121                .context("disk admission path missing")?;
122            let device = std::fs::metadata(probe)?.dev();
123            if !devices.insert(device) {
124                continue;
125            }
126            let open = |path: &Path| -> Result<File> {
127                Ok(OpenOptions::new()
128                    .read(true)
129                    .write(true)
130                    .create(true)
131                    .truncate(false)
132                    .custom_flags(libc::O_NOFOLLOW)
133                    .open(path)?)
134            };
135            // Explicitly unlock on return. A concurrent fork can briefly
136            // inherit this descriptor before exec closes it; closing only the
137            // parent's copy would leave the namespace spuriously busy.
138            let admission = Self {
139                _file: open(&dir.join(format!("{device}.lock")))?,
140                _activity: false,
141            };
142            let started = std::time::Instant::now();
143            loop {
144                match crate::pr_head::try_lock_file(&admission._file) {
145                    Ok(()) => break,
146                    Err(error)
147                        if matches!(
148                            error.kind(),
149                            std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
150                        ) && started.elapsed() < std::time::Duration::from_secs(1) =>
151                    {
152                        std::thread::sleep(std::time::Duration::from_millis(5));
153                    }
154                    Err(error) => {
155                        return Err(error)
156                            .context("disk admission busy or unavailable; retry without dispatch");
157                    }
158                }
159            }
160            let mut active = 0u64;
161            let mut available = None;
162            for slot in 0..1024 {
163                let file = dir.join(format!("{device}-{slot}.reserve"));
164                if !file.exists() {
165                    if available.is_none() {
166                        available = Some(file);
167                    }
168                    continue;
169                }
170                let file = open(&file)?;
171                match crate::pr_head::try_lock_file(&file) {
172                    Ok(()) if available.is_none() => {
173                        available = Some(dir.join(format!("{device}-{slot}.reserve")))
174                    }
175                    Ok(()) => {}
176                    Err(_) => active = active.saturating_add(RESERVE),
177                }
178            }
179            let free = free_bytes(probe)?;
180            let required = min_free.saturating_add(active).saturating_add(RESERVE);
181            ensure!(
182                free >= required,
183                "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
184            );
185            let file = open(&available.context("disk admission reservation slots exhausted")?)?;
186            crate::pr_head::try_lock_file(&file)?;
187            reservations.push(Self {
188                _file: file,
189                _activity: false,
190            });
191        }
192        Ok(reservations)
193    }
194
195    #[cfg(not(unix))]
196    pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
197        // Registered seats already fail closed without Unix containment.
198        Ok(Vec::new())
199    }
200
201    pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
202        let out = std::process::Command::new("git")
203            .args(["rev-parse", "--absolute-git-dir"])
204            .current_dir(worktree)
205            .quiet()
206            .output()?;
207        ensure!(
208            out.status.success(),
209            "cannot identify output lease directory"
210        );
211        let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
212        Self::lock(&dir.join("magi-output.lock"), exclusive)
213    }
214    fn lock(path: &Path, exclusive: bool) -> Result<Self> {
215        let file = OpenOptions::new()
216            .create(true)
217            .truncate(false)
218            .read(true)
219            .write(true)
220            .open(path)
221            .with_context(|| format!("output lease: open {}", path.display()))?;
222        #[cfg(unix)]
223        {
224            use std::os::fd::AsRawFd;
225            let mode = if exclusive {
226                libc::LOCK_EX
227            } else {
228                libc::LOCK_SH
229            };
230            // SAFETY: live file descriptor, no retained pointers.
231            ensure!(
232                unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
233                "output lease: outputs are in use or being reclaimed: {}",
234                path.display()
235            );
236        }
237        #[cfg(not(unix))]
238        let _ = exclusive;
239        if !exclusive {
240            // Empty legacy/maintenance-only leases carry no activity witness.
241            file.set_len(1)?;
242            file.set_modified(std::time::SystemTime::now())
243                .with_context(|| format!("record output activity {}", path.display()))?;
244        }
245        Ok(Self {
246            _file: file,
247            _activity: !exclusive,
248        })
249    }
250}
251#[cfg(unix)]
252impl Drop for OutputLease {
253    fn drop(&mut self) {
254        use std::os::fd::AsRawFd;
255        if self._activity
256            && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
257        {
258            // If release time cannot be recorded, retain a nonempty invalid
259            // witness rather than trusting the old acquisition timestamp.
260            let _ = self._file.set_len(2);
261            tracing::warn!("could not record output lease release activity: {error}");
262        }
263        // SAFETY: unlock this guard's live descriptor, including inherited copies.
264        unsafe {
265            libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
266        }
267    }
268}
269
270#[derive(Default, Debug, Serialize, Deserialize)]
271#[serde(default)]
272pub(crate) struct Report {
273    pub checked_at: Option<Timestamp>,
274    pub limit_bytes: u64,
275    pub reclaim_budget_bytes: u64,
276    /// Logical inventory, including protected and unproven data.
277    pub measured_bytes: u64,
278    /// Logical bytes unlinked, not a measurement of physical disk recovery.
279    pub freed_bytes: u64,
280    pub measured_allocated_file_bytes: Option<u64>,
281    /// Candidates proven under locks in this pass; uninspected data is excluded.
282    pub proven_reclaimable_bytes: u64,
283    /// Estimated allocated blocks recoverable if every proven link is removed.
284    /// Inodes with links outside the candidate set contribute zero.
285    pub proven_reclaimable_allocated_bytes: Option<u64>,
286    pub protected_or_unproven_bytes: u64,
287    /// Deduplicated allocated file blocks whose final link was unlinked. Open
288    /// descriptors and shared extents can prevent an equivalent physical gain.
289    pub estimated_released_allocated_bytes: Option<u64>,
290    /// Observed free-space change on the home volume; concurrent writes, clones
291    /// and deferred reclamation mean this is not attributable to this sweep.
292    pub observed_home_free_change_bytes: Option<i64>,
293    pub retired: usize,
294    pub evicted_copies: usize,
295    pub removed_empty_roots: usize,
296    pub source_unlinked_logical_bytes: u64,
297    pub source_unlinked_allocated_bytes: Option<u64>,
298    pub source_evidence_added_logical_bytes: u64,
299    pub source_evidence_added_allocated_bytes: Option<u64>,
300    pub retained: Vec<String>,
301    pub failures: Vec<String>,
302}
303
304#[cfg(unix)]
305fn allocated_inventory(
306    root: &Path,
307    seen: &mut std::collections::BTreeSet<(u64, u64)>,
308) -> Result<u64> {
309    use std::os::unix::fs::MetadataExt;
310    ensure!(
311        plain_dir(root) && root.canonicalize()? == root,
312        "allocated inventory root is not canonical"
313    );
314    let mut dirs = vec![root.to_owned()];
315    let mut bytes = 0u64;
316    while let Some(dir) = dirs.pop() {
317        for entry in std::fs::read_dir(dir)? {
318            let path = entry?.path();
319            let metadata = std::fs::symlink_metadata(&path)?;
320            if metadata.is_dir() {
321                dirs.push(path);
322            } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
323                bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
324            }
325        }
326    }
327    Ok(bytes)
328}
329
330#[derive(Default)]
331struct ReleasedAllocation {
332    #[cfg(unix)]
333    files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
334    #[cfg(unix)]
335    removed: std::collections::BTreeMap<(u64, u64), u64>,
336}
337
338impl ReleasedAllocation {
339    fn of(paths: &[PathBuf]) -> Result<Self> {
340        #[cfg(unix)]
341        {
342            use std::os::unix::fs::MetadataExt;
343            let mut files = std::collections::BTreeMap::new();
344            for path in paths {
345                let m = std::fs::symlink_metadata(path)?;
346                ensure!(m.is_file(), "allocation candidate is not regular");
347                files.insert(
348                    path.clone(),
349                    (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
350                );
351            }
352            Ok(Self {
353                files,
354                removed: Default::default(),
355            })
356        }
357        #[cfg(not(unix))]
358        {
359            let _ = paths;
360            Ok(Self::default())
361        }
362    }
363    fn reclaimable_allocated_bytes(&self) -> Option<u64> {
364        #[cfg(unix)]
365        {
366            let mut inodes = std::collections::BTreeMap::new();
367            for &(device, inode, links, blocks) in self.files.values() {
368                let entry = inodes
369                    .entry((device, inode))
370                    .or_insert((0u64, links, blocks));
371                entry.0 += 1;
372            }
373            Some(
374                inodes
375                    .values()
376                    .filter(|(count, links, _)| count == links)
377                    .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
378            )
379        }
380        #[cfg(not(unix))]
381        {
382            None
383        }
384    }
385
386    fn unlinked(&mut self, path: &Path) -> Option<u64> {
387        #[cfg(unix)]
388        {
389            let (device, inode, links, blocks) = self.files[path];
390            let removed = self.removed.entry((device, inode)).or_default();
391            *removed += 1;
392            Some(if *removed == links { blocks } else { 0 })
393        }
394        #[cfg(not(unix))]
395        {
396            let _ = path;
397            None
398        }
399    }
400}
401
402fn plain_dir(path: &Path) -> bool {
403    std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
404}
405
406/// Old run records outlive their checkout and output directories. Only an
407/// absent path means nothing to reclaim; inaccessible paths remain errors.
408fn run_paths_missing(state: &RunState) -> Result<bool> {
409    for path in [state.worktree_root(), state.repo.clone()] {
410        match std::fs::metadata(&path) {
411            Ok(_) => {}
412            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
413            Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
414        }
415    }
416    Ok(false)
417}
418
419/// Enumerate actual registered worktrees, not just candidates in run.json.
420async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
421    let root = state.worktree_root();
422    if !plain_dir(&root) {
423        return Ok(Vec::new());
424    }
425    let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
426    registered_worktrees(state, &list)
427}
428
429fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
430    let root = state.worktree_root();
431    if !plain_dir(&root) {
432        return Ok(Vec::new());
433    }
434    let root = root.canonicalize()?;
435    let mut paths = Vec::new();
436    for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
437        let path = PathBuf::from(path);
438        if plain_dir(&path)
439            && path.parent() == Some(root.as_path())
440            && path.canonicalize()? == path
441            && path.join(".git").is_file()
442        {
443            paths.push(path);
444        }
445    }
446    Ok(paths)
447}
448
449/// Recognize Cargo's generated directory, never merely an ignored directory.
450/// Unknown root entries are retained; cache identity/profile paths fail closed.
451async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
452    validate_cargo_target(worktree, false).await
453}
454
455async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
456    let target = worktree.join("target");
457    ensure!(plain_dir(&target), "target absent or not a plain directory");
458    ensure!(target.canonicalize()? == target, "target path changed");
459    ensure!(
460        git::git(worktree, &["ls-files", "--", "target"])
461            .await?
462            .is_empty(),
463        "tracked target contents"
464    );
465    ensure!(
466        git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
467            .await?
468            .ok(),
469        "target is not ignored"
470    );
471    cache_tag(&target, allow_missing_tag)?;
472    let path = target.join(".rustc_info.json");
473    plain_file(&path, "rustc identity")?;
474    let info: serde_json::Value = serde_json::from_slice(
475        &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
476    )
477    .with_context(|| format!("parse rustc identity {}", path.display()))?;
478    ensure!(
479        info.get("rustc_fingerprint").is_some(),
480        "unknown Cargo identity {}",
481        path.display()
482    );
483    if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
484        ensure!(
485            info.get("rustc_fingerprint")
486                .is_some_and(serde_json::Value::is_u64)
487                && info
488                    .get("outputs")
489                    .and_then(serde_json::Value::as_object)
490                    .is_some_and(|outputs| outputs.values().any(|output| {
491                        output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
492                            && output
493                                .get("stdout")
494                                .and_then(serde_json::Value::as_str)
495                                .is_some_and(|stdout| stdout.starts_with("rustc "))
496                            && output
497                                .get("stderr")
498                                .is_some_and(serde_json::Value::is_string)
499                    }))
500                && info
501                    .get("successes")
502                    .is_some_and(serde_json::Value::is_object),
503            "missing-tag rustc identity structure invalid: {}",
504            path.display()
505        );
506        let mut profiles = 0;
507        for profile in ["debug", "release"] {
508            let dir = target.join(profile);
509            if !dir.exists() {
510                continue;
511            }
512            for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
513                ensure!(
514                    plain_dir(&subdir) && subdir.canonicalize()? == subdir,
515                    "missing-tag profile proof: {} is not a plain directory",
516                    subdir.display()
517                );
518            }
519            plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
520            profiles += 1;
521        }
522        ensure!(
523            profiles > 0,
524            "missing-tag proof: no Cargo profile in {}",
525            target.display()
526        );
527    }
528    for name in ["debug", "release"] {
529        let path = target.join(name);
530        match std::fs::symlink_metadata(&path) {
531            Ok(metadata) => ensure!(
532                metadata.is_dir() && path.canonicalize()? == path,
533                "profile validation: not a plain directory: {}",
534                path.display()
535            ),
536            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
537            Err(error) => {
538                return Err(error)
539                    .with_context(|| format!("profile validation: inspect {}", path.display()));
540            }
541        }
542    }
543    Ok(target)
544}
545
546fn plain_file(path: &Path, stage: &str) -> Result<()> {
547    let meta = std::fs::symlink_metadata(path)
548        .with_context(|| format!("{stage}: inspect {}", path.display()))?;
549    ensure!(
550        meta.is_file() && path.canonicalize()? == path,
551        "{stage}: not a plain file: {}",
552        path.display()
553    );
554    Ok(())
555}
556
557fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
558    let path = target.join("CACHEDIR.TAG");
559    match std::fs::symlink_metadata(&path) {
560        Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
561            return Ok(());
562        }
563        result => {
564            result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
565        }
566    }
567    plain_file(&path, "cache tag")?;
568    let tag = std::fs::read_to_string(&path)
569        .with_context(|| format!("cache tag: read {}", path.display()))?;
570    ensure!(
571        tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
572        "unknown Cargo cache tag: {}",
573        path.display()
574    );
575    Ok(())
576}
577
578/// Observe compiler metadata and artifacts under the output and Cargo locks.
579/// Unknown contents never become deletion authority. Unreadable timestamps or
580/// a future timestamp retain outputs, even when the configured grace is zero.
581fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
582    let mut dirs = vec![target.to_owned()];
583    let mut entries = 0;
584    while let Some(dir) = dirs.pop() {
585        for entry in std::fs::read_dir(&dir)
586            .with_context(|| format!("output activity: list {}", dir.display()))?
587        {
588            let path = entry?.path();
589            entries += 1;
590            ensure!(
591                entries <= 100_000,
592                "output activity inventory too large: {}",
593                target.display()
594            );
595            let metadata = std::fs::symlink_metadata(&path)
596                .with_context(|| format!("output activity: inspect {}", path.display()))?;
597            if metadata.is_dir() {
598                // Only supported Cargo profiles supply execution evidence.
599                if dir != target
600                    || matches!(
601                        path.file_name().and_then(|s| s.to_str()),
602                        Some("debug" | "release")
603                    )
604                {
605                    dirs.push(path);
606                }
607            } else if metadata.is_file()
608                && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
609            {
610                let at =
611                    Timestamp::try_from(metadata.modified().with_context(|| {
612                        format!("output activity: modified {}", path.display())
613                    })?)?;
614                ensure!(
615                    crate::clean::due(now, at, grace),
616                    "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
617                    path.display()
618                );
619            }
620        }
621    }
622    Ok(())
623}
624
625pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
626    use std::io::Read as _;
627    let mut header = [0; 8];
628    let n = File::open(path)?.read(&mut header)?;
629    let header = &header[..n];
630    Ok(header.starts_with(b"\x7fELF")
631        || header.starts_with(b"!<arch>\n")
632        || header.starts_with(b"rust\0\0\0")
633        || header.starts_with(b"MZ\x90\0")
634        || [
635            [0xfe, 0xed, 0xfa, 0xce],
636            [0xce, 0xfa, 0xed, 0xfe],
637            [0xfe, 0xed, 0xfa, 0xcf],
638            [0xcf, 0xfa, 0xed, 0xfe],
639            [0xca, 0xfe, 0xba, 0xbe],
640            [0xca, 0xfe, 0xba, 0xbf],
641        ]
642        .iter()
643        .any(|magic| header.starts_with(magic)))
644}
645
646/// Legacy tags identify a cache, not ownership of every file inside it.
647/// Only direct rustc outputs named by dep-info may be reclaimed. Build-script
648/// output, incremental caches, unknown files and the manifests themselves stay.
649fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
650    let mut outputs = std::collections::BTreeSet::new();
651    for profile in ["debug", "release"] {
652        for dir in [target.join(profile), target.join(profile).join("deps")] {
653            if !plain_dir(&dir) || dir.canonicalize()? != dir {
654                continue;
655            }
656            for entry in std::fs::read_dir(&dir)? {
657                let path = entry?.path();
658                if path.extension().is_none_or(|ext| ext != "d")
659                    || !std::fs::symlink_metadata(&path)?.is_file()
660                    || std::fs::metadata(&path)?.len() > 1024 * 1024
661                    || path.canonicalize()? != path
662                {
663                    continue;
664                }
665                let body = std::fs::read_to_string(&path)?;
666                for line in body.lines() {
667                    let Some((names, dependencies)) = line.split_once(": ") else {
668                        continue;
669                    };
670                    if dependencies.trim().is_empty() {
671                        continue;
672                    }
673                    // Rustc escapes spaces and backslashes in Make paths.
674                    // Unsupported Make syntax fails closed for that record.
675                    let mut words = Vec::new();
676                    let mut word = String::new();
677                    let mut escaped = false;
678                    for c in names.chars() {
679                        if escaped {
680                            word.push(c);
681                            escaped = false;
682                        } else if c == '\\' {
683                            escaped = true;
684                        } else if c.is_whitespace() {
685                            if !word.is_empty() {
686                                words.push(std::mem::take(&mut word));
687                            }
688                        } else {
689                            word.push(c);
690                        }
691                    }
692                    if escaped || word.contains(['$', '%']) {
693                        continue;
694                    }
695                    if !word.is_empty() {
696                        words.push(word);
697                    }
698                    for name in words {
699                        let output = PathBuf::from(name);
700                        let stem = path.file_stem().unwrap_or_default().to_string_lossy();
701                        let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
702                        let artifact_name =
703                            output_stem == stem || output_stem == format!("lib{stem}");
704                        let artifact_kind = output.extension().is_none_or(|ext| {
705                            matches!(
706                                ext.to_str(),
707                                Some(
708                                    "rlib"
709                                        | "rmeta"
710                                        | "so"
711                                        | "dylib"
712                                        | "a"
713                                        | "lib"
714                                        | "dll"
715                                        | "exe"
716                                        | "pdb"
717                                )
718                            )
719                        });
720                        if artifact_name
721                            && artifact_kind
722                            && output.parent() == Some(dir.as_path())
723                            && output.extension().is_none_or(|ext| ext != "d")
724                            && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
725                            && output.canonicalize().is_ok_and(|p| p == output)
726                            && artifact_signature(&output)?
727                        {
728                            outputs.insert(output);
729                        }
730                    }
731                }
732            }
733        }
734    }
735    Ok(outputs.into_iter().collect())
736}
737
738#[derive(Serialize, Deserialize)]
739struct CargoResidue {
740    target: PathBuf,
741    files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
742}
743
744// Keep the exact metadata as historical evidence. An old proof cannot adopt
745// newly added files, changed metadata, symlinks or a different worktree.
746fn residue_files(
747    target: &Path,
748    outputs: &[PathBuf],
749) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
750    let mut files = std::collections::BTreeMap::new();
751    let mut dirs = vec![target.to_owned()];
752    let mut bytes = 0usize;
753    let mut entries = 0usize;
754    while let Some(dir) = dirs.pop() {
755        for entry in std::fs::read_dir(dir)? {
756            let path = entry?.path();
757            entries += 1;
758            ensure!(entries <= 4096, "residue inventory too large");
759            let metadata = std::fs::symlink_metadata(&path)?;
760            ensure!(
761                !metadata.file_type().is_symlink(),
762                "symlink in Cargo residue"
763            );
764            if metadata.is_dir() {
765                dirs.push(path);
766            } else {
767                ensure!(metadata.is_file(), "non-file in Cargo residue");
768                if outputs.contains(&path) {
769                    continue;
770                }
771                ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
772                let body = std::fs::read(&path)?;
773                bytes = bytes.saturating_add(body.len());
774                ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
775                files.insert(path.strip_prefix(target)?.to_owned(), body);
776            }
777        }
778    }
779    Ok(files)
780}
781
782fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
783    Ok(home
784        .join("runs")
785        .join(id)
786        .join("build-residue")
787        .join(worktree.file_name().context("missing worktree name")?))
788}
789
790// Artifact eviction tolerates unrelated root data. Whole-target removal must
791// still reject it, including empty directories absent from the file inventory.
792fn residue_root(target: &Path) -> Result<()> {
793    for entry in std::fs::read_dir(target)? {
794        let name = entry?.file_name();
795        ensure!(
796            matches!(
797                name.to_str(),
798                Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
799            ),
800            "unrecognized Cargo residue root entry"
801        );
802    }
803    Ok(())
804}
805
806fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
807    cache_tag(target, false)?;
808    residue_root(target)?;
809    let files = residue_files(target, outputs)?;
810    for (relative, body) in &files {
811        let path = target.join(relative);
812        let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
813        let lock = matches!(
814            relative.to_str(),
815            Some("debug/.cargo-lock" | "release/.cargo-lock")
816        ) && body.is_empty();
817        let dep_info = path.extension().is_some_and(|ext| ext == "d")
818            && outputs.iter().any(|output| {
819                output.parent() == path.parent()
820                    && (output.file_stem() == path.file_stem()
821                        || output.file_stem().is_some_and(|stem| {
822                            stem.to_string_lossy()
823                                == format!(
824                                    "lib{}",
825                                    path.file_stem().unwrap_or_default().to_string_lossy()
826                                )
827                        }))
828            });
829        ensure!(identity || lock || dep_info, "unproven Cargo residue");
830    }
831    let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
832    std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
833    let tmp = proof.with_extension("tmp");
834    std::fs::write(
835        &tmp,
836        serde_json::to_vec(&CargoResidue {
837            target: target.to_owned(),
838            files,
839        })?,
840    )?;
841    std::fs::rename(tmp, proof)?;
842    Ok(())
843}
844
845pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
846    let target = cargo_target(worktree).await?;
847    residue_root(&target)?;
848    let locks = cargo_locks(&target)?;
849    let proof: CargoResidue =
850        serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
851    ensure!(
852        proof.target == target && proof.files == residue_files(&target, &[])?,
853        "Cargo residue changed or contains unproven files"
854    );
855    Ok(locks)
856}
857
858/// Supplemental exclusion for Cargo builds launched outside the managed seats.
859pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
860    let mut held = Vec::new();
861    for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
862        [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
863    }) {
864        let lock = target.join(profile).join(name);
865        match std::fs::symlink_metadata(&lock) {
866            Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
867            result => {
868                result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
869            }
870        }
871        {
872            plain_file(&lock, "Cargo lock")?;
873            let file = OpenOptions::new()
874                .read(true)
875                .write(true)
876                .open(&lock)
877                .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
878            crate::pr_head::try_lock_file(&file)
879                .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
880            held.push(file);
881        }
882    }
883    Ok(held)
884}
885
886/// All repositories share one cap and one per-pass deletion budget. The report
887/// survives a restart and explains every skipped run/target or failed operation.
888pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
889    let home = home.to_owned();
890    let disk = disk.clone();
891    let runtime = tokio::runtime::Handle::current();
892    tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
893        .await
894        .unwrap_or_else(|e| Report {
895            failures: vec![format!("retention worker failed: {e}")],
896            ..Report::default()
897        })
898}
899
900#[derive(Default, Serialize, Deserialize)]
901struct Measurement {
902    at: Option<Timestamp>,
903    bytes: u64,
904}
905
906async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
907    let mut report = Report {
908        checked_at: Some(now),
909        limit_bytes: disk.worktree_limit_bytes,
910        reclaim_budget_bytes: disk.worktree_reclaim_bytes,
911        ..Report::default()
912    };
913    if !cfg!(unix) {
914        report
915            .failures
916            .push("Automatic output reclamation requires Unix process containment".into());
917        return report;
918    }
919    let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
920        return report;
921    };
922    let mut targets = Vec::new();
923    let mut under_pressure = false;
924    let mut registrations = std::collections::HashMap::new();
925    let measurements_path = home.join("output-measurements.json");
926    let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
927        std::fs::read(&measurements_path)
928            .ok()
929            .and_then(|b| serde_json::from_slice(&b).ok())
930            .unwrap_or_default();
931    let mut observed = std::collections::BTreeSet::new();
932    let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
933        .into_iter()
934        .flatten()
935        .flatten()
936        .map(|e| e.file_name().to_string_lossy().into_owned())
937        .collect();
938    ids.sort();
939    for id in ids {
940        if !crate::run::is_run_id(&id) {
941            continue;
942        }
943        let state = match RunState::load_under(&id, home) {
944            Ok(state) => state,
945            Err(e) => {
946                report
947                    .retained
948                    .push(format!("{id}: unreadable run ownership: {e:#}"));
949                continue;
950            }
951        };
952        if disk.worktree_limit_bytes == 0 {
953            continue;
954        }
955        match run_paths_missing(&state) {
956            Ok(true) => continue,
957            Ok(false) => {}
958            Err(e) => {
959                report.failures.push(format!("{id}: {e:#}"));
960                continue;
961            }
962        }
963        let list = match registrations.entry(state.repo.clone()) {
964            std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
965            std::collections::hash_map::Entry::Vacant(entry) => {
966                match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
967                    Ok(list) => entry.insert(list),
968                    Err(e) => {
969                        // The checkout may disappear after the inventory check.
970                        // Do not confuse a missing Git executable with that case.
971                        if matches!(run_paths_missing(&state), Ok(true)) {
972                            continue;
973                        }
974                        report.failures.push(format!("{id}: {e:#}"));
975                        continue;
976                    }
977                }
978            }
979        };
980        let paths = match registered_worktrees(&state, list) {
981            Ok(paths) => paths,
982            Err(e) => {
983                report.failures.push(format!("{id}: {e:#}"));
984                continue;
985            }
986        };
987        for path in paths {
988            if !path.join("target").exists() {
989                continue;
990            }
991            let target = path.join("target");
992            observed.insert(target.clone());
993            let pressure =
994                crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
995            under_pressure |= pressure;
996            let measurement = measurements.entry(target).or_default();
997            if pressure
998                || measurement
999                    .at
1000                    .is_none_or(|at| crate::clean::due(now, at, 300))
1001            {
1002                measurement.bytes = crate::disk::dir_size(&path.join("target"));
1003                measurement.at = Some(now);
1004            }
1005            let size = measurement.bytes;
1006            report.measured_bytes = report.measured_bytes.saturating_add(size);
1007            targets.push((state.output_activity(), id.clone(), path, size));
1008        }
1009    }
1010    if under_pressure {
1011        // Inventory sizes prioritize scarce-space recovery only; deletion authority
1012        // and budget accounting come from declared outputs measured under locks.
1013        targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1014    } else {
1015        targets.sort_by_key(|t| t.0);
1016    }
1017    let free_before = crate::disk::free_bytes(home).ok();
1018    #[cfg(unix)]
1019    {
1020        let mut seen = std::collections::BTreeSet::new();
1021        report.measured_allocated_file_bytes = Some(0);
1022        report.estimated_released_allocated_bytes = Some(0);
1023        report.proven_reclaimable_allocated_bytes = Some(0);
1024        for (_, _, path, _) in &targets {
1025            report.measured_allocated_file_bytes = report
1026                .measured_allocated_file_bytes
1027                .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1028                .map(|(a, b)| a.saturating_add(b));
1029        }
1030    }
1031    let mut remaining = report.measured_bytes;
1032    let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1033    let mut deletion_started = false;
1034    let mut oversized_started = false;
1035    for (updated, id, path, _) in targets {
1036        if disk.worktree_limit_bytes == 0
1037            || remaining <= disk.worktree_limit_bytes
1038                && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1039        {
1040            report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1041            continue;
1042        }
1043        if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1044            report
1045                .retained
1046                .push(format!("{id}: per-pass byte budget spent"));
1047            continue;
1048        }
1049        let result = async {
1050            ensure!(
1051                crate::clean::due(now, updated, disk.fold_grace_secs),
1052                "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1053            );
1054            let _owner = crate::pr_head::owner_lock(home, &id)?;
1055            let state = RunState::load_under(&id, home)?;
1056            ensure!(
1057                state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1058                    == Liveness::Dead,
1059                "run live or ownership unknown"
1060            );
1061            ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1062            let _driver = crate::driver::maintenance(home, &id)?;
1063            let _outputs = OutputLease::take(&path, true)?;
1064            ensure!(
1065                worktrees(&state).await?.contains(&path),
1066                "worktree registration changed"
1067            );
1068            let target = validate_cargo_target(&path, true).await
1069                .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1070            let _cargo = cargo_locks(&target)
1071                .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1072            // Cargo may have finished between identity inspection and locking.
1073            // Revalidate the same authority with all writers excluded.
1074            validate_cargo_target(&path, true).await
1075                .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1076            let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1077            if lease_meta.len() > 0 {
1078                ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1079                let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1080                ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1081                    "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1082            }
1083            output_age(&target, now, disk.fold_grace_secs)
1084                .with_context(|| format!("stage output-age: {}", target.display()))?;
1085            let mut outputs = declared_outputs(&target)
1086                .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1087            let incremental = match crate::incremental::candidates(&path) {
1088                Ok(candidates) => candidates,
1089                Err(error) => {
1090                    report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1091                    crate::incremental::Reclaim::default()
1092                }
1093            };
1094            let incremental_files = incremental.files.clone();
1095            outputs.extend(incremental_files.iter().cloned());
1096            outputs.sort();
1097            outputs.dedup();
1098            ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1099            let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1100                .with_context(|| format!("stage measure: {}", p.display())))
1101                .collect::<Result<Vec<_>>>()?;
1102            let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1103            report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1104            let mut allocation = ReleasedAllocation::of(&outputs)?;
1105            report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1106                .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1107            let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1108            // 予算は1回の削除量を抑えるもので、大きな出力を永久に保持するためではない。
1109            // 正の予算では最初の削除対象だけ超過を許し、その後は予算消費済みとする。
1110            // 途中で失敗しても再度許可しないよう、例外は削除開始時に消費する。
1111            ensure!(!deletion_started || !exceeds_budget,
1112                "declared outputs exceed remaining per-pass byte budget");
1113            // Record adoption before deletion; interrupted deletion is revalidated
1114            // on the next pass. No source file or branch is modified.
1115            let ledger = home.join("runs").join(&id).join("build-outputs.json");
1116            std::fs::write(
1117                &ledger,
1118                serde_json::to_vec(
1119                    &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1120                )?,
1121            ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1122            // Unknown leftovers still permit artifact-only eviction, but never
1123            // authorize retirement. Only a complete proven residue is adopted.
1124            if !incremental_files.is_empty() {
1125                report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1126            } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1127                report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1128            }
1129            ensure!(worktrees(&state).await?.contains(&path),
1130                "worktree registration changed before deletion");
1131            deletion_started = true;
1132            oversized_started = exceeds_budget;
1133            measurements.remove(&target);
1134            for (output, size) in outputs.iter().zip(sizes) {
1135                std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1136                report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1137                    .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1138                // Preserve actual progress even if a later removal fails.
1139                report.freed_bytes = report.freed_bytes.saturating_add(size);
1140                remaining = remaining.saturating_sub(size);
1141            }
1142            incremental.finish().context("finish incremental reclamation")?;
1143            report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1144            Ok::<_, anyhow::Error>(())
1145        }
1146        .await;
1147        if let Err(e) = result {
1148            report.retained.push(format!(
1149                "{id}/{}: {e:#}",
1150                path.file_name().unwrap_or_default().to_string_lossy()
1151            ));
1152        }
1153    }
1154    measurements.retain(|path, _| observed.contains(path));
1155    if let Ok(body) = serde_json::to_vec(&measurements)
1156        && let Err(e) = std::fs::write(&measurements_path, body)
1157    {
1158        report.failures.push(format!("output measurements: {e}"));
1159    }
1160    copies::evict(home, disk, now, &mut report).await;
1161    retire_empty(home, disk, now, &mut report).await;
1162    report.protected_or_unproven_bytes = report
1163        .measured_bytes
1164        .saturating_sub(report.proven_reclaimable_bytes);
1165    let pressure_remaining = pressure_paths
1166        .iter()
1167        .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1168    if !report.failures.is_empty()
1169        || disk.worktree_limit_bytes > 0
1170            && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1171    {
1172        let details = report
1173            .retained
1174            .iter()
1175            .filter(|reason| {
1176                !reason.contains("within policy")
1177                    && !reason.ends_with("unmanifested files and Cargo metadata retained")
1178            })
1179            .take(4)
1180            .chain(report.failures.iter().take(2))
1181            .cloned()
1182            .collect::<Vec<_>>()
1183            .join("; ");
1184        crate::notices::raise_in(
1185            home,
1186            crate::notices::Notice::warn(
1187                "disk:retention",
1188                format!(
1189                    "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1190                    report.measured_bytes,
1191                    report.proven_reclaimable_bytes,
1192                    report.protected_or_unproven_bytes,
1193                    report.freed_bytes,
1194                    if report.proven_reclaimable_bytes == 0 {
1195                        "No inspected contents have deletion authority."
1196                    } else {
1197                        "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1198                    }
1199                ),
1200            ),
1201        );
1202    }
1203    report.observed_home_free_change_bytes = free_before
1204        .zip(crate::disk::free_bytes(home).ok())
1205        .map(|(before, after)| {
1206            (i128::from(after) - i128::from(before))
1207                .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1208        });
1209    let store = home.join("retention.json");
1210    if let Ok(body) = serde_json::to_vec_pretty(&report) {
1211        let tmp = home.join("retention.json.tmp");
1212        if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1213            tracing::warn!("retention report could not be saved: {e}");
1214        }
1215    }
1216    tracing::info!(
1217        measured_bytes = report.measured_bytes,
1218        freed_bytes = report.freed_bytes,
1219        retired = report.retired,
1220        retained = report.retained.len(),
1221        failures = report.failures.len(),
1222        "retained worktree cleanup"
1223    );
1224    for reason in report.retained.iter().chain(&report.failures) {
1225        tracing::debug!(%reason, "worktree retained");
1226    }
1227    report
1228}
1229
1230async fn retire_empty(
1231    home: &Path,
1232    disk: &crate::config::Disk,
1233    now: Timestamp,
1234    report: &mut Report,
1235) {
1236    if !disk.auto_fold {
1237        return;
1238    }
1239    let queue = crate::queue::Queue::at(home.join("queue"));
1240    let questions = crate::ask::Questions::at(home.join("questions"));
1241    let open = questions.list();
1242    for listed in queue.list() {
1243        let Ok(_claim) = queue.claim(&listed.id) else {
1244            continue;
1245        };
1246        let Ok(task) = queue.get(&listed.id) else {
1247            continue;
1248        };
1249        if open
1250            .iter()
1251            .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1252        {
1253            report.retained.push(format!(
1254                "{}: task has an unanswered owner decision",
1255                task.id
1256            ));
1257            continue;
1258        }
1259        let sources: Vec<_> = task
1260            .runs
1261            .iter()
1262            .collect::<std::collections::BTreeSet<_>>()
1263            .into_iter()
1264            .cloned()
1265            .collect();
1266        let mut useful = Vec::new();
1267        let mut selected = Vec::new();
1268        let mut public = Vec::new();
1269        let mut unreadable = false;
1270        for id in &sources {
1271            let Ok(state) = RunState::load_under(id, home) else {
1272                unreadable = true;
1273                continue;
1274            };
1275            if state.released() || state.retired_to.is_some() {
1276                continue;
1277            }
1278            if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1279                selected.push(state.clone());
1280            }
1281            if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1282                public.push(state.clone());
1283            }
1284            if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1285                useful.push(state);
1286            }
1287        }
1288        // Prefer a durable owner-approved selection, then one existing PR
1289        // owner, then an unambiguous useful source. Distinct historical changes
1290        // without an owner selection cannot be resolved by creation time.
1291        let pool = if !selected.is_empty() {
1292            selected
1293        } else if !public.is_empty() {
1294            public
1295        } else {
1296            useful
1297        };
1298        if unreadable || pool.len() != 1 {
1299            report.retained.push(format!(
1300                "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1301                task.id,
1302                sources.len()
1303            ));
1304            continue;
1305        }
1306        let authority = &pool[0];
1307        if authority.candidates.is_empty() {
1308            continue;
1309        }
1310        let latest = &authority.id;
1311        for id in &sources {
1312            if id == latest || report.retired >= 5 {
1313                continue;
1314            }
1315            if let Ok(state) = RunState::load_under(id, home)
1316                && state.retired_to.as_ref() == Some(latest)
1317                && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1318            {
1319                continue;
1320            }
1321            let result = async {
1322                ensure!(
1323                    !task
1324                        .resume_override
1325                        .as_ref()
1326                        .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1327                    "operator-pinned recovery run"
1328                );
1329                ensure!(
1330                    !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1331                    "selected replacement recovery source"
1332                );
1333                let _owner = crate::pr_head::owner_lock(home, id)?;
1334                let mut state = RunState::load_under(id, home)?;
1335                ensure!(
1336                    state.repo == authority.repo,
1337                    "repository differs from recovery authority"
1338                );
1339                ensure!(
1340                    !crate::pr_recovery::explicit_sources_retained(home, id),
1341                    "explicitly retained recovery source"
1342                );
1343                ensure!(
1344                    state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1345                        == Liveness::Dead,
1346                    "run live or ownership unknown"
1347                );
1348                let _driver = crate::driver::maintenance(home, id)?;
1349                ensure!(
1350                    crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1351                    "within grace period"
1352                );
1353                ensure!(
1354                    !open.iter().any(|q| q.run == *id && q.status.open()),
1355                    "unanswered owner decision"
1356                );
1357                ensure!(
1358                    state.pr.is_none()
1359                        && state.pr_head.is_none()
1360                        && state.pr_recovery.is_none()
1361                        && state.land_approval.is_none()
1362                        && state.merge_decision.is_none()
1363                        && state.reviews.is_empty()
1364                        && state.recovery.is_none()
1365                        && state.candidate_preservation.is_empty()
1366                        && state.interrupted_implementations.is_empty()
1367                        && !state.released(),
1368                    "PR ownership or recovery checkpoint"
1369                );
1370                let proof = prove_empty(home, &state, latest).await?;
1371                commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1372                // Durable retirement precedes removal. A crash retries removal
1373                // using the same proof; it cannot make this attempt resumable.
1374                for path in &proof.paths {
1375                    ensure!(
1376                        git::worktree_remove_clean(&state.repo, path).await?,
1377                        "Git refused clean worktree removal"
1378                    );
1379                }
1380                Ok::<_, anyhow::Error>(())
1381            }
1382            .await;
1383            match result {
1384                Ok(()) => report.retired += 1,
1385                Err(e) => report.retained.push(format!("{id}: {e:#}")),
1386            }
1387        }
1388    }
1389}
1390
1391struct EmptyProof {
1392    paths: Vec<PathBuf>,
1393    _outputs: Vec<OutputLease>,
1394    _cargo: Vec<File>,
1395}
1396
1397// Compare actual source bytes with the original tree independently of index
1398// stat caches, assume-unchanged flags, sparse checkout and fsmonitor hints.
1399pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1400    let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1401    let mut files = Vec::new();
1402    let mut blobs = Vec::new();
1403    for record in tree.split('\0').filter(|s| !s.is_empty()) {
1404        let (header, name) = record.split_once('\t').context("unknown tree record")?;
1405        let parts: Vec<_> = header.split_whitespace().collect();
1406        ensure!(
1407            parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1408            "unsupported tracked source kind"
1409        );
1410        let path = worktree.join(name);
1411        let meta = std::fs::symlink_metadata(&path)?;
1412        ensure!(
1413            meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1414            "noncanonical tracked source"
1415        );
1416        #[cfg(unix)]
1417        {
1418            use std::os::unix::fs::PermissionsExt as _;
1419            ensure!(
1420                (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1421                "tracked executable mode differs from original base"
1422            );
1423        }
1424        files.push(path.to_str().context("non UTF-8 source")?.to_owned());
1425        blobs.push(parts[2].to_owned());
1426    }
1427    for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1428        let mut args = vec!["hash-object", "--no-filters", "--"];
1429        args.extend(paths.iter().map(String::as_str));
1430        let actual = git::git(worktree, &args).await?;
1431        ensure!(
1432            actual.lines().eq(expected.iter().map(String::as_str)),
1433            "tracked bytes differ from original base"
1434        );
1435    }
1436    Ok(())
1437}
1438
1439async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1440    let id = &state.id;
1441    let paths = worktrees(state).await?;
1442    let mut guards = Vec::new();
1443    let mut cargo_guards = Vec::new();
1444    for path in &paths {
1445        guards.push(OutputLease::take(path, true)?);
1446        ensure!(
1447            git::rev_parse(path, "HEAD").await? == state.base_commit,
1448            "unique or unrecorded commits"
1449        );
1450        ensure!(
1451            git::git(
1452                path,
1453                &[
1454                    "--no-optional-locks",
1455                    "status",
1456                    "--porcelain",
1457                    "--untracked-files=all"
1458                ]
1459            )
1460            .await?
1461            .is_empty(),
1462            "tracked or untracked source changes"
1463        );
1464        ensure!(
1465            git::git(
1466                path,
1467                &[
1468                    "--no-optional-locks",
1469                    "status",
1470                    "--porcelain",
1471                    "--untracked-files=all",
1472                    "--ignored",
1473                    "--",
1474                    ".",
1475                    ":(exclude)target"
1476                ]
1477            )
1478            .await?
1479            .is_empty(),
1480            "source changes, ignored files or build outputs"
1481        );
1482        tracked_base(path, &state.base_commit).await?;
1483        match std::fs::symlink_metadata(path.join("target")) {
1484            Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1485            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1486            Err(error) => return Err(error.into()),
1487        }
1488        for checkpoint in [
1489            "rebase-merge",
1490            "rebase-apply",
1491            "sequencer",
1492            "BISECT_LOG",
1493            "REVERT_HEAD",
1494            "AUTO_MERGE",
1495            "MERGE_MSG",
1496            "CHERRY_PICK_HEAD",
1497            "MERGE_HEAD",
1498            "REBASE_HEAD",
1499        ] {
1500            let location = git::git(
1501                path,
1502                &[
1503                    "rev-parse",
1504                    "--path-format=absolute",
1505                    "--git-path",
1506                    checkpoint,
1507                ],
1508            )
1509            .await?;
1510            ensure!(
1511                matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1512                "retained Git recovery metadata: {checkpoint}"
1513            );
1514        }
1515        // Git also initializes ORIG_HEAD when creating a worktree.
1516        // Only an origin outside the base is a recovery source.
1517        let origin_path = git::git(
1518            path,
1519            &[
1520                "rev-parse",
1521                "--path-format=absolute",
1522                "--git-path",
1523                "ORIG_HEAD",
1524            ],
1525        )
1526        .await?;
1527        match std::fs::symlink_metadata(origin_path.trim()) {
1528            Ok(meta) => {
1529                ensure!(
1530                    meta.is_file() && !meta.file_type().is_symlink(),
1531                    "unknown ORIG_HEAD artifact"
1532                );
1533                git::rev_parse(path, "ORIG_HEAD")
1534                    .await
1535                    .context("unreadable ORIG_HEAD")?;
1536            }
1537            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1538            Err(e) => return Err(e.into()),
1539        }
1540        if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1541            ensure!(
1542                git::git_raw(
1543                    path,
1544                    &["merge-base", "--is-ancestor", &original, &state.base_commit]
1545                )
1546                .await?
1547                .ok(),
1548                "ORIG_HEAD retains commits outside the original base"
1549            );
1550        }
1551        let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1552        for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1553            ensure!(
1554                git::git_raw(
1555                    path,
1556                    &["merge-base", "--is-ancestor", head, &state.base_commit]
1557                )
1558                .await?
1559                .ok(),
1560                "worktree reflog retains commits outside the original base"
1561            );
1562        }
1563        for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1564            ensure!(
1565                !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1566                    .await?
1567                    .ok(),
1568                "unfinished Git operation"
1569            );
1570        }
1571    }
1572    // Missing worktrees cannot establish that unrecorded prep edits
1573    // were empty. Recorded candidates must also point at the base.
1574    if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1575        return Ok(EmptyProof {
1576            paths,
1577            _outputs: guards,
1578            _cargo: cargo_guards,
1579        });
1580    }
1581    ensure!(!paths.is_empty(), "no worktrees to prove empty");
1582    for candidate in &state.candidates {
1583        ensure!(
1584            (paths.contains(&candidate.worktree)
1585                || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1586                && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1587            "candidate recovery source"
1588        );
1589    }
1590    let pattern = format!("refs/heads/magi/{}/", state.short());
1591    let branches = git::git(
1592        &state.repo,
1593        &["for-each-ref", "--format=%(objectname)", &pattern],
1594    )
1595    .await?;
1596    ensure!(
1597        branches.lines().all(|head| head == state.base_commit),
1598        "unrecorded candidate branch changes"
1599    );
1600    Ok(EmptyProof {
1601        paths,
1602        _outputs: guards,
1603        _cargo: cargo_guards,
1604    })
1605}
1606
1607/// The only task-write critical section in retirement. All Git proof and
1608/// deletion happens outside it, under the run's owner/driver/output leases.
1609/// Atomic task writes make the earlier snapshot consistent; this final compare
1610/// rejects every intervening owner selection, hold, edit or history update.
1611fn commit_retirement(
1612    home: &Path,
1613    queue: &crate::queue::Queue,
1614    expected: &crate::queue::Task,
1615    state: &mut RunState,
1616    authority: &str,
1617    sources: &[String],
1618    now: Timestamp,
1619) -> Result<()> {
1620    let _write = queue.lock_task(&expected.id)?;
1621    let current = queue.get(&expected.id)?;
1622    ensure!(
1623        serde_json::to_value(&current)? == serde_json::to_value(expected)?,
1624        "task changed during retirement proof; retaining worktrees for the current owner decision"
1625    );
1626    ensure!(
1627        !current
1628            .resume_override
1629            .as_ref()
1630            .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1631            && !current
1632                .replacement
1633                .as_ref()
1634                .is_some_and(|pin| pin.run == state.id),
1635        "current owner selected this recovery source"
1636    );
1637    ensure!(
1638        !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1639        "explicitly retained recovery source"
1640    );
1641    ensure!(
1642        !crate::ask::Questions::at(home.join("questions"))
1643            .list()
1644            .iter()
1645            .any(|q| q.status.open()
1646                && (q.run == state.id
1647                    || q.run == current.id
1648                    || current.blocked_by.contains(&q.id))),
1649        "owner decision arrived during retirement proof"
1650    );
1651    let dir = home.join("recovery-lineages");
1652    std::fs::create_dir_all(&dir)?;
1653    let tmp = dir.join(format!("{}.tmp", current.id));
1654    let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1655        "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1656    std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1657    std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1658    state.retired_to = Some(authority.to_owned());
1659    state.event(
1660        "retention",
1661        format!(
1662            "empty attempt retired to {authority}; recovery sources: {}",
1663            sources.join(", ")
1664        ),
1665    );
1666    state.save_under(home)?;
1667    // No Git command or worktree removal may be added before this guard drops.
1668    Ok(())
1669}
1670
1671/// Wait for the emergency reserve to be exhausted. The normal floor requests
1672/// a checkpoint first; half the floor stops owned writers before node completion.
1673/// A failed measurement also closes admission rather than allowing blind writes.
1674pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1675    if min_free == 0 {
1676        return std::future::pending().await;
1677    }
1678    loop {
1679        for path in &paths {
1680            let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1681            match crate::disk::free_bytes(existing) {
1682                Ok(free) if free >= min_free / 2 => {}
1683                Ok(free) => {
1684                    return format!(
1685                        "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1686                    );
1687                }
1688                Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1689            }
1690        }
1691        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1692    }
1693}
1694
1695#[cfg(all(test, unix))]
1696mod tests {
1697    use super::*;
1698    use crate::{
1699        config::{Config, Disk},
1700        queue::{Source, Task},
1701        run::{Candidate, RunStatus},
1702    };
1703
1704    struct Fx {
1705        _temp: tempfile::TempDir,
1706        home: PathBuf,
1707        repo: PathBuf,
1708        bay: PathBuf,
1709        base: String,
1710    }
1711    impl Fx {
1712        async fn new() -> Self {
1713            let temp = tempfile::tempdir().unwrap();
1714            let root = temp.path().canonicalize().unwrap();
1715            let home = root.join("home");
1716            let repo = root.join("repo");
1717            let bay = root.join("worktrees");
1718            std::fs::create_dir_all(&repo).unwrap();
1719            std::fs::create_dir_all(home.join("runs")).unwrap();
1720            git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1721            git::git(&repo, &["config", "user.name", "Fixture"])
1722                .await
1723                .unwrap();
1724            git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1725                .await
1726                .unwrap();
1727            git::git(&repo, &["config", "commit.gpgsign", "false"])
1728                .await
1729                .unwrap();
1730            std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1731            std::fs::write(repo.join("source"), "base\n").unwrap();
1732            git::git(&repo, &["add", "."]).await.unwrap();
1733            git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1734            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1735            Self {
1736                _temp: temp,
1737                home,
1738                repo,
1739                bay,
1740                base,
1741            }
1742        }
1743        async fn run(&self, tag: &str, useful: bool) -> RunState {
1744            let mut cfg = Config::default();
1745            cfg.graph.worktree_root = Some(self.bay.clone());
1746            let mut state = RunState::new(
1747                self.repo.clone(),
1748                "main".into(),
1749                self.base.clone(),
1750                "task".into(),
1751                cfg,
1752            );
1753            state.id = format!("20260101-000000-{tag}");
1754            state.status = RunStatus::Blocked;
1755            state.driver_exited = true;
1756            let wt = state.worktree_root().join("cand-A");
1757            let branch = format!("magi/{tag}/A");
1758            git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1759                .await
1760                .unwrap();
1761            if useful {
1762                std::fs::write(wt.join("source"), tag).unwrap();
1763                git::git(&wt, &["add", "."]).await.unwrap();
1764                git::git(&wt, &["commit", "-m", "candidate change"])
1765                    .await
1766                    .unwrap();
1767            }
1768            state.candidates.push(Candidate {
1769                index: 0,
1770                label: 'A',
1771                agent: "fixture".into(),
1772                branch,
1773                worktree: wt,
1774                summary: String::new(),
1775                stat: String::new(),
1776                files: usize::from(useful),
1777                commits: usize::from(useful),
1778                empty: !useful,
1779                failed: None,
1780                verified_noop: None,
1781                duration_ms: 0,
1782                folded: false,
1783            });
1784            state.save_under(&self.home).unwrap();
1785            state
1786        }
1787        fn disk() -> Disk {
1788            Disk {
1789                worktree_limit_bytes: 1,
1790                worktree_reclaim_bytes: 1024 * 1024,
1791                fold_grace_secs: 0,
1792                min_free_bytes: 0,
1793                ..Disk::default()
1794            }
1795        }
1796        fn target(wt: &Path) {
1797            let target = wt.join("target");
1798            std::fs::create_dir_all(target.join("debug")).unwrap();
1799            let mut artifact = [1; 200];
1800            artifact[..4].copy_from_slice(b"\x7fELF");
1801            std::fs::write(target.join("debug/output"), artifact).unwrap();
1802            let output = target
1803                .join("debug/output")
1804                .display()
1805                .to_string()
1806                .replace('\\', "\\\\")
1807                .replace(' ', "\\ ");
1808            std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1809            std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1810            std::fs::write(
1811                target.join("CACHEDIR.TAG"),
1812                "Signature: 8a477f597d28d172789f06886806bc55\n",
1813            )
1814            .unwrap();
1815            std::fs::write(
1816                target.join(".rustc_info.json"),
1817                "{\"rustc_fingerprint\":123}",
1818            )
1819            .unwrap();
1820        }
1821        fn now() -> Timestamp {
1822            Timestamp::now() + jiff::SignedDuration::from_secs(2)
1823        }
1824        fn task(&self, runs: &[RunState]) -> Task {
1825            let mut task = Task::new(
1826                "fixture".into(),
1827                "task".into(),
1828                self.repo.clone(),
1829                Source::Human,
1830            );
1831            task.runs = runs.iter().map(|s| s.id.clone()).collect();
1832            crate::queue::Queue::at(self.home.join("queue"))
1833                .put(&mut task)
1834                .unwrap();
1835            task
1836        }
1837    }
1838
1839    #[cfg(unix)]
1840    #[tokio::test]
1841    async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1842        let f = Fx::new().await;
1843        let state = f.run("ca01", true).await;
1844        let wt = &state.candidates[0].worktree;
1845        let generation = crate::incremental::tests::generate(wt);
1846        // Remove dep-info authority: incremental proof must be independent.
1847        for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1848            for entry in std::fs::read_dir(dir).unwrap() {
1849                let path = entry.unwrap().path();
1850                if path.extension().is_some_and(|x| x == "d") {
1851                    std::fs::remove_file(path).unwrap();
1852                }
1853            }
1854        }
1855        assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1856        assert_eq!(
1857            sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1858            0,
1859            "live generation is protected"
1860        );
1861        drop(generation);
1862        let candidates = crate::incremental::candidates(wt).unwrap();
1863        let logical: u64 = candidates
1864            .files
1865            .iter()
1866            .map(|p| std::fs::metadata(p).unwrap().len())
1867            .sum();
1868        let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1869        std::fs::write(&unknown, "retain unknown data").unwrap();
1870        drop(candidates);
1871        let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1872        assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1873        assert!(
1874            refused
1875                .retained
1876                .iter()
1877                .any(|s| s.contains("provenance mismatch"))
1878        );
1879        std::fs::remove_file(&unknown).unwrap();
1880        let legacy = wt.join("target/debug/incremental/legacy-session");
1881        std::fs::create_dir(&legacy).unwrap();
1882        // Even compiler-authored bytes do not retroactively confer ownership.
1883        let candidates = crate::incremental::candidates(wt).unwrap();
1884        let bin = candidates
1885            .files
1886            .iter()
1887            .find(|p| p.extension().is_some_and(|x| x == "bin"))
1888            .unwrap();
1889        std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1890        drop(candidates);
1891        let cargo = cargo_locks(&wt.join("target")).unwrap();
1892        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1893        drop(cargo);
1894        let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1895        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1896        drop(outputs);
1897        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1898        assert_eq!(report.freed_bytes, logical, "{report:?}");
1899        assert_eq!(report.proven_reclaimable_bytes, logical);
1900        assert!(report.measured_bytes > logical);
1901        assert_eq!(
1902            report.protected_or_unproven_bytes,
1903            report.measured_bytes - logical
1904        );
1905        #[cfg(unix)]
1906        {
1907            assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
1908            assert!(
1909                report.measured_allocated_file_bytes.unwrap()
1910                    >= report.estimated_released_allocated_bytes.unwrap()
1911            );
1912        }
1913        assert!(report.observed_home_free_change_bytes.is_some());
1914        assert!(legacy.join("query-cache.bin").is_file());
1915        assert!(wt.join("src/lib.rs").is_file());
1916        assert_eq!(report.retired, 0);
1917    }
1918
1919    #[cfg(unix)]
1920    #[tokio::test]
1921    async fn incremental_and_deps_links_account_for_retained_third_link() {
1922        use std::os::unix::fs::MetadataExt;
1923        let f = Fx::new().await;
1924        let state = f.run("ca02", true).await;
1925        let wt = &state.candidates[0].worktree;
1926        let generation = crate::incremental::tests::generate(wt);
1927        drop(generation);
1928        let candidates = crate::incremental::candidates(wt).unwrap();
1929        let objects = candidates
1930            .files
1931            .iter()
1932            .filter(|p| p.extension().is_some_and(|x| x == "o"))
1933            .take(2)
1934            .cloned()
1935            .collect::<Vec<_>>();
1936        assert_eq!(
1937            objects.len(),
1938            2,
1939            "fixture must produce real incremental objects"
1940        );
1941        let mut outputs = candidates.files.clone();
1942        drop(candidates);
1943        let target = wt.join("target");
1944        for dir in [target.join("debug"), target.join("debug/deps")] {
1945            for entry in std::fs::read_dir(dir).unwrap() {
1946                let path = entry.unwrap().path();
1947                if path.extension().is_some_and(|x| x == "d") {
1948                    std::fs::remove_file(path).unwrap();
1949                }
1950            }
1951        }
1952        // Cargo also leaves hard-linked object outputs in deps. Remove those
1953        // fixture outputs so the only external links are the ones below.
1954        for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
1955            let path = entry.unwrap().path();
1956            if path.is_file() {
1957                std::fs::remove_file(path).unwrap();
1958            }
1959        }
1960        for (index, object) in objects.iter().enumerate() {
1961            assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
1962            let dep = target.join(format!("debug/deps/link{index}"));
1963            std::fs::hard_link(object, &dep).unwrap();
1964            std::fs::write(
1965                dep.with_extension("d"),
1966                format!("{}: src/lib.rs\n", dep.display()),
1967            )
1968            .unwrap();
1969            outputs.push(dep);
1970        }
1971        let retained = wt.join("retained-input");
1972        std::fs::hard_link(&objects[1], &retained).unwrap();
1973        let sentinel = std::fs::read(&retained).unwrap();
1974        let logical: u64 = outputs
1975            .iter()
1976            .map(|p| std::fs::metadata(p).unwrap().len())
1977            .sum();
1978        let mut seen = std::collections::BTreeSet::new();
1979        let retained_meta = std::fs::metadata(&retained).unwrap();
1980        let retained_inode = (retained_meta.dev(), retained_meta.ino());
1981        let mut expected = 0u64;
1982        for output in &outputs {
1983            let m = std::fs::metadata(output).unwrap();
1984            let inode = (m.dev(), m.ino());
1985            if inode != retained_inode && seen.insert(inode) {
1986                expected += m.blocks() * 512;
1987            }
1988        }
1989        assert!(expected > 0);
1990        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1991        assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
1992        assert_eq!(report.freed_bytes, logical, "{report:?}");
1993        assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
1994        assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
1995        assert!(outputs.iter().all(|p| !p.exists()));
1996        assert_eq!(std::fs::read(retained).unwrap(), sentinel);
1997        assert!(wt.join("src/lib.rs").is_file());
1998    }
1999
2000    #[tokio::test]
2001    async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2002        let f = Fx::new().await;
2003        let state = f.run("ca03", true).await;
2004        let wt = &state.candidates[0].worktree;
2005        Fx::target(wt);
2006        std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2007        let unknown = wt.join("target/debug/operator-note");
2008        std::fs::write(&unknown, vec![1; 4096]).unwrap();
2009        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2010        assert_eq!(report.proven_reclaimable_bytes, 0);
2011        #[cfg(unix)]
2012        assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2013        assert!(report.protected_or_unproven_bytes >= 4096);
2014        let message = || {
2015            crate::notices::Notices::at(f.home.join("notifications"))
2016                .list()
2017                .into_iter()
2018                .find(|n| n.key == "disk:retention")
2019                .unwrap()
2020                .message
2021        };
2022        let notice = message();
2023        assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2024        assert!(notice.contains("protected or unproven"), "{notice}");
2025        assert!(
2026            notice.contains("legacy incremental/build and unknown files retained"),
2027            "{notice}"
2028        );
2029        let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2030        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2031        assert_eq!(report.freed_bytes, 0);
2032        let notice = message();
2033        assert!(notice.contains(&state.id), "{notice}");
2034        assert!(
2035            notice.contains("lock") || notice.contains("owner"),
2036            "{notice}"
2037        );
2038        drop(owner);
2039        assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2040    }
2041
2042    #[tokio::test]
2043    async fn real_cargo_build_holds_the_target_against_reclamation() {
2044        let f = Fx::new().await;
2045        let state = f.run("ca02", true).await;
2046        let wt = &state.candidates[0].worktree;
2047        Fx::target(wt);
2048        std::fs::create_dir(wt.join("src")).unwrap();
2049        std::fs::write(
2050            wt.join("Cargo.toml"),
2051            "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2052        )
2053        .unwrap();
2054        std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2055        std::fs::write(
2056            wt.join("build.rs"),
2057            r#"fn main() {
2058            let ready = std::env::var("FIXTURE_READY").unwrap();
2059            let release = std::env::var("FIXTURE_RELEASE").unwrap();
2060            std::fs::write(ready, "ready").unwrap();
2061            let start = std::time::Instant::now();
2062            while !std::path::Path::new(&release).exists() && start.elapsed().as_secs() < 15 {
2063                std::thread::sleep(std::time::Duration::from_millis(10));
2064            }
2065        }"#,
2066        )
2067        .unwrap();
2068        let ready = wt.join("ready");
2069        let release = wt.join("release");
2070        let mut cargo = tokio::process::Command::new("cargo");
2071        cargo.env_clear();
2072        for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2073            if let Some(value) = std::env::var_os(key) {
2074                cargo.env(key, value);
2075            }
2076        }
2077        let mut child = cargo
2078            .current_dir(wt)
2079            .env("RUSTUP_TOOLCHAIN", "stable")
2080            .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2081            .env("CARGO_INCREMENTAL", "1")
2082            .env("FIXTURE_READY", &ready)
2083            .env("FIXTURE_RELEASE", &release)
2084            .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2085            .arg(wt.join("target"))
2086            .stdout(std::process::Stdio::null())
2087            .stderr(std::process::Stdio::null())
2088            .kill_on_drop(true)
2089            .spawn()
2090            .unwrap();
2091        let arrived = tokio::time::timeout(std::time::Duration::from_secs(10), async {
2092            while !ready.exists() {
2093                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2094            }
2095        })
2096        .await
2097        .is_ok();
2098        let report = if arrived {
2099            Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2100        } else {
2101            None
2102        };
2103        // Release and join our build before making assertions, including on failure.
2104        std::fs::write(release, "release").unwrap();
2105        let status = tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await;
2106        if status.is_err() {
2107            child.kill().await.unwrap();
2108            child.wait().await.unwrap();
2109        }
2110        assert!(
2111            arrived,
2112            "fixture build never reached its active build script"
2113        );
2114        assert!(status.unwrap().unwrap().success());
2115        let report = report.unwrap();
2116        assert_eq!(report.freed_bytes, 0, "{report:?}");
2117        assert!(
2118            report
2119                .retained
2120                .iter()
2121                .any(|s| s.contains("Cargo build is active")),
2122            "{report:?}"
2123        );
2124        assert!(wt.join("target/debug/output").is_file());
2125        assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2126    }
2127
2128    #[cfg(unix)]
2129    #[test]
2130    fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2131        let temp = tempfile::tempdir().unwrap();
2132        let a = temp.path().join("a");
2133        let b = temp.path().join("b");
2134        let outside = temp.path().join("retained-input");
2135        std::fs::write(&a, vec![1; 65536]).unwrap();
2136        std::fs::hard_link(&a, &b).unwrap();
2137        std::fs::hard_link(&a, &outside).unwrap();
2138        let mut seen = Default::default();
2139        let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2140        assert!(blocks > 0);
2141        let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2142        std::fs::remove_file(&a).unwrap();
2143        assert_eq!(accounting.unlinked(&a), Some(0));
2144        std::fs::remove_file(&b).unwrap();
2145        assert_eq!(accounting.unlinked(&b), Some(0));
2146        assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2147        let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2148        std::fs::remove_file(&outside).unwrap();
2149        assert_eq!(accounting.unlinked(&outside), Some(blocks));
2150    }
2151
2152    #[cfg(unix)]
2153    #[test]
2154    fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2155        let temp = tempfile::tempdir_in(".").unwrap();
2156        let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2157        assert!(home.is_relative());
2158        let paths = vec![std::env::current_dir().unwrap()];
2159        let gib = 1024 * 1024 * 1024;
2160        let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2161        assert!(
2162            OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2163                9 * gib
2164            ))
2165            .is_err()
2166        );
2167        drop(reservation);
2168        assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2169        let bad = temp.path().join("symlink-state");
2170        std::fs::create_dir(&bad).unwrap();
2171        std::os::unix::fs::symlink(
2172            home.canonicalize().unwrap().join("disk-admission"),
2173            bad.join("disk-admission"),
2174        )
2175        .unwrap();
2176        assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2177    }
2178
2179    #[cfg(unix)]
2180    #[test]
2181    fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2182        let temp = tempfile::tempdir().unwrap();
2183        let home = temp.path().canonicalize().unwrap();
2184        let paths = vec![home.clone(), home.join("not-created-yet")];
2185        let gib = 1024 * 1024 * 1024;
2186        let floor = 8 * gib;
2187        let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2188        assert_eq!(first.len(), 1);
2189        let blocked =
2190            OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2191        assert!(blocked.to_string().contains("active writer reservations"));
2192        let second =
2193            OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2194        drop(first);
2195        assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2196        drop(second);
2197        assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2198        assert!(
2199            OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2200        );
2201    }
2202
2203    #[cfg(unix)]
2204    #[test]
2205    fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2206        let temp = tempfile::tempdir().unwrap();
2207        let home = temp.path().canonicalize().unwrap();
2208        let barrier = std::sync::Barrier::new(2);
2209        let gib = 1024 * 1024 * 1024;
2210        let results = std::thread::scope(|scope| {
2211            let threads: Vec<_> = (0..2)
2212                .map(|_| {
2213                    scope.spawn(|| {
2214                        barrier.wait();
2215                        OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2216                            Ok(100 + gib)
2217                        })
2218                    })
2219                })
2220                .collect();
2221            threads
2222                .into_iter()
2223                .map(|thread| thread.join().unwrap())
2224                .collect::<Vec<_>>()
2225        });
2226        assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2227        assert!(
2228            results
2229                .iter()
2230                .filter_map(|r| r.as_ref().err())
2231                .all(|e| e.to_string().contains("active writer reservations"))
2232        );
2233    }
2234
2235    #[tokio::test]
2236    async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2237        real_cargo_fixture(false).await;
2238    }
2239
2240    #[tokio::test]
2241    async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2242        real_cargo_fixture(true).await;
2243    }
2244
2245    async fn real_cargo_fixture(precreate: bool) {
2246        let f = Fx::new().await;
2247        let state = f.run("ca00", true).await;
2248        let wt = &state.candidates[0].worktree;
2249        let source = "fn main() { println!(\"retained fixture\"); }\n";
2250        std::fs::create_dir(wt.join("src")).unwrap();
2251        std::fs::write(wt.join("src/main.rs"), source).unwrap();
2252        std::fs::write(
2253            wt.join("Cargo.toml"),
2254            "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2255        )
2256        .unwrap();
2257        if precreate {
2258            std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2259        }
2260        let mut cargo = std::process::Command::new("cargo");
2261        cargo.env_clear();
2262        for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2263            if let Some(value) = std::env::var_os(key) {
2264                cargo.env(key, value);
2265            }
2266        }
2267        let build = cargo
2268            .current_dir(wt)
2269            .env("RUSTUP_TOOLCHAIN", "stable")
2270            .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2271            .env("CARGO_INCREMENTAL", "0")
2272            .args([
2273                "build",
2274                "--offline",
2275                "--release",
2276                "--jobs",
2277                "1",
2278                "--target-dir",
2279            ])
2280            .arg(wt.join("target"))
2281            .output()
2282            .unwrap();
2283        assert!(
2284            build.status.success(),
2285            "{}",
2286            String::from_utf8_lossy(&build.stderr)
2287        );
2288        assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2289        if precreate {
2290            // Cargo leaves pre-existing target roots untagged on subsequent builds too.
2291            let again = cargo.output().unwrap();
2292            assert!(
2293                again.status.success(),
2294                "{}",
2295                String::from_utf8_lossy(&again.stderr)
2296            );
2297            assert!(!wt.join("target/CACHEDIR.TAG").exists());
2298            assert!(cargo_target(wt).await.is_err());
2299        }
2300        git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2301            .await
2302            .unwrap();
2303        git::git(wt, &["commit", "-m", "real Cargo fixture"])
2304            .await
2305            .unwrap();
2306        let artifact = wt.join("target/release/retained-fixture");
2307        assert!(artifact.is_file());
2308        let unknown = wt.join("target/release/operator-note");
2309        std::fs::write(&unknown, "keep this note\n").unwrap();
2310
2311        let target = wt.join("target");
2312        let mut preserved = std::collections::BTreeMap::new();
2313        for relative in [
2314            "tmp/sentinel",
2315            "magi-tools/sentinel",
2316            "lifecycle-client-check.log",
2317        ] {
2318            let path = target.join(relative);
2319            std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2320            std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2321            preserved.insert(path.clone(), std::fs::read(path).unwrap());
2322        }
2323        // Even genuine compiler bytes and valid dep-info outside supported
2324        // profiles cannot expand the deletion inventory.
2325        let decoy = target.join("magi-tools/retained-fixture");
2326        std::fs::copy(&artifact, &decoy).unwrap();
2327        let dep_info = decoy.with_extension("d");
2328        std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2329        for path in [
2330            decoy,
2331            dep_info,
2332            wt.join("Cargo.toml"),
2333            wt.join("Cargo.lock"),
2334        ] {
2335            preserved.insert(path.clone(), std::fs::read(path).unwrap());
2336        }
2337        let proven = declared_outputs(&target).unwrap();
2338
2339        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2340        assert!(out.freed_bytes > 0, "{out:?}");
2341        for (path, bytes) in preserved {
2342            assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2343        }
2344        assert!(!artifact.exists(), "{out:?}");
2345        assert_eq!(
2346            std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2347            source
2348        );
2349        assert_eq!(
2350            std::fs::read_to_string(&unknown).unwrap(),
2351            "keep this note\n"
2352        );
2353        assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2354        let ledger: serde_json::Value = serde_json::from_slice(
2355            &std::fs::read(
2356                f.home
2357                    .join("runs")
2358                    .join(&state.id)
2359                    .join("build-outputs.json"),
2360            )
2361            .unwrap(),
2362        )
2363        .unwrap();
2364        assert_eq!(
2365            ledger["identity"],
2366            if precreate {
2367                "rustc-profile-dep-info"
2368            } else {
2369                "cache-tag"
2370            }
2371        );
2372        if precreate {
2373            assert!(!target.join("CACHEDIR.TAG").exists());
2374            assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2375        }
2376        let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2377        assert!(deleted.contains(&artifact));
2378        assert_eq!(deleted, proven);
2379        for path in deleted {
2380            assert!(!path.exists());
2381            assert!(
2382                [target.join("release"), target.join("release/deps")]
2383                    .contains(&path.parent().unwrap().to_owned())
2384            );
2385        }
2386        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2387    }
2388
2389    fn missing_tag_target(wt: &Path) {
2390        Fx::target(wt);
2391        let target = wt.join("target");
2392        std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2393        for subdir in ["debug/deps", "debug/.fingerprint"] {
2394            std::fs::create_dir_all(target.join(subdir)).unwrap();
2395        }
2396        std::fs::write(target.join(".rustc_info.json"),
2397            r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2398    }
2399
2400    #[tokio::test]
2401    async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2402        for legacy in [false, true] {
2403            let f = Fx::new().await;
2404            let mut state = f.run("ac00", true).await;
2405            Fx::target(&state.candidates[0].worktree);
2406            let original = state.updated_at;
2407            let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2408            if legacy {
2409                let path = f.home.join("runs").join(&state.id).join("run.json");
2410                let mut value = serde_json::to_value(&state).unwrap();
2411                value.as_object_mut().unwrap().remove("output_activity_at");
2412                std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2413                state = RunState::load_under(&state.id, &f.home).unwrap();
2414                assert_eq!(state.output_activity(), original);
2415            }
2416            let age = state.output_activity();
2417            for offset in [30, 20, 10] {
2418                state
2419                    .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2420                    .unwrap();
2421                state = RunState::load_under(&state.id, &f.home).unwrap();
2422                assert_eq!(state.output_activity(), age);
2423            }
2424            let mut disk = Fx::disk();
2425            disk.fold_grace_secs = 3600;
2426            let report = sweep(&f.home, &disk, now).await;
2427            assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2428        }
2429    }
2430
2431    #[tokio::test]
2432    async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2433        for evidence in [
2434            "execution",
2435            "artifact",
2436            "dep-info",
2437            "fingerprint",
2438            "lease",
2439            "future",
2440        ] {
2441            let f = Fx::new().await;
2442            let mut state = f.run("ac01", true).await;
2443            let wt = state.candidates[0].worktree.clone();
2444            missing_tag_target(&wt);
2445            let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2446            let recent = now - jiff::SignedDuration::from_secs(10);
2447            match evidence {
2448                "execution" => {
2449                    state.note_output_activity(recent);
2450                    state.save_under_at(&f.home, recent).unwrap();
2451                }
2452                "lease" => {
2453                    let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2454                    let file = lease._file.try_clone().unwrap();
2455                    drop(lease);
2456                    file.set_modified(std::time::SystemTime::from(recent))
2457                        .unwrap();
2458                }
2459                _ => {
2460                    let relative = match evidence {
2461                        "dep-info" => "debug/output.d",
2462                        "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2463                        _ => "debug/output",
2464                    };
2465                    let path = wt.join("target").join(relative);
2466                    let file = OpenOptions::new()
2467                        .write(true)
2468                        .create(true)
2469                        .truncate(false)
2470                        .open(path)
2471                        .unwrap();
2472                    let at = if evidence == "future" {
2473                        now + jiff::SignedDuration::from_secs(10)
2474                    } else {
2475                        recent
2476                    };
2477                    file.set_modified(std::time::SystemTime::from(at)).unwrap();
2478                }
2479            }
2480            let mut disk = Fx::disk();
2481            disk.fold_grace_secs = 3600;
2482            let report = sweep(&f.home, &disk, now).await;
2483            assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2484            assert!(
2485                report.retained.iter().any(|s| s.contains("grace period")
2486                    || s.contains("execution activity after inventory")),
2487                "{evidence}: {report:?}"
2488            );
2489            let aged = now + jiff::SignedDuration::from_secs(7200);
2490            let report = sweep(&f.home, &disk, aged).await;
2491            assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2492        }
2493    }
2494
2495    #[tokio::test]
2496    async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2497        let f = Fx::new().await;
2498        let state = f.run("ac02", true).await;
2499        let wt = &state.candidates[0].worktree;
2500        missing_tag_target(wt);
2501        let target = wt.join("target");
2502        let unknown = target.join("debug/notes");
2503        std::fs::write(&unknown, "operator data").unwrap();
2504        for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2505            let path = target.join("debug").join(lock);
2506            let file = OpenOptions::new()
2507                .create(true)
2508                .truncate(false)
2509                .read(true)
2510                .write(true)
2511                .open(&path)
2512                .unwrap();
2513            crate::pr_head::try_lock_file(&file).unwrap();
2514            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2515            assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2516            assert!(
2517                report
2518                    .retained
2519                    .iter()
2520                    .any(|s| s.contains(path.to_str().unwrap())),
2521                "{report:?}"
2522            );
2523        }
2524        let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2525        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2526        drop(owner);
2527        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2528        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2529        drop(driver);
2530        let output = OutputLease::borrow(wt).unwrap().unwrap();
2531        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2532        drop(output);
2533        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2534        assert_eq!(report.freed_bytes, 200, "{report:?}");
2535        assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2536        assert!(!target.join("CACHEDIR.TAG").exists());
2537        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2538        assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2539        assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2540    }
2541
2542    #[tokio::test]
2543    async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2544        let f = Fx::new().await;
2545        let state = f.run("ac04", true).await;
2546        let wt = &state.candidates[0].worktree;
2547        missing_tag_target(wt);
2548        let target = wt.join("target");
2549        let first = target.join("debug/output");
2550        let second = target.join("debug/second");
2551        std::fs::copy(&first, &second).unwrap();
2552        std::fs::write(
2553            second.with_extension("d"),
2554            format!("{}: source\n", second.display()),
2555        )
2556        .unwrap();
2557        let outputs = declared_outputs(&target).unwrap();
2558        assert_eq!(outputs.len(), 2);
2559        // Simulate interruption after durable adoption and one successful unlink.
2560        std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2561            serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2562        std::fs::remove_file(first).unwrap();
2563        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2564        assert_eq!(report.freed_bytes, 200, "{report:?}");
2565        assert!(!second.exists());
2566        assert!(second.with_extension("d").exists());
2567        assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2568    }
2569
2570    #[tokio::test]
2571    async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2572        for invalid in [
2573            "identity",
2574            "deps",
2575            "fingerprint",
2576            "lock",
2577            "dep-info",
2578            "tag-symlink",
2579            "bad-tag",
2580        ] {
2581            let f = Fx::new().await;
2582            let state = f.run("ac03", true).await;
2583            let wt = &state.candidates[0].worktree;
2584            missing_tag_target(wt);
2585            let target = wt.join("target");
2586            match invalid {
2587                "identity" => std::fs::write(
2588                    target.join(".rustc_info.json"),
2589                    r#"{"rustc_fingerprint":123}"#,
2590                )
2591                .unwrap(),
2592                "deps" | "fingerprint" => {
2593                    let path = target.join("debug").join(if invalid == "deps" {
2594                        "deps"
2595                    } else {
2596                        ".fingerprint"
2597                    });
2598                    std::fs::remove_dir(&path).unwrap();
2599                    std::os::unix::fs::symlink(&f.repo, path).unwrap();
2600                }
2601                "lock" => {
2602                    std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2603                }
2604                "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2605                "tag-symlink" => {
2606                    std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2607                        .unwrap()
2608                }
2609                "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2610                _ => unreachable!(),
2611            }
2612            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2613            assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2614            assert!(target.join("debug/output").is_file());
2615        }
2616    }
2617
2618    #[tokio::test]
2619    async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2620        let f = Fx::new().await;
2621        let first = f.run("ba00", true).await;
2622        let second = f.run("ba01", true).await;
2623        for state in [&first, &second] {
2624            Fx::target(&state.candidates[0].worktree);
2625        }
2626        let mut disk = Fx::disk();
2627        disk.worktree_reclaim_bytes = 100;
2628        let out = sweep(&f.home, &disk, Fx::now()).await;
2629        assert_eq!(out.freed_bytes, 200, "{out:?}");
2630        assert!(
2631            out.retained
2632                .contains(&format!("{}: per-pass byte budget spent", second.id)),
2633            "{out:?}"
2634        );
2635        assert!(
2636            !first.candidates[0]
2637                .worktree
2638                .join("target/debug/output")
2639                .exists()
2640        );
2641        assert!(
2642            second.candidates[0]
2643                .worktree
2644                .join("target/debug/output")
2645                .exists()
2646        );
2647        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2648    }
2649
2650    #[tokio::test]
2651    async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2652        let f = Fx::new().await;
2653        let small = f.run("bb00", true).await;
2654        let large = f.run("bb01", true).await;
2655        for state in [&small, &large] {
2656            Fx::target(&state.candidates[0].worktree);
2657        }
2658        let output = large.candidates[0].worktree.join("target/debug/output");
2659        std::fs::OpenOptions::new()
2660            .write(true)
2661            .open(&output)
2662            .unwrap()
2663            .set_len(1000)
2664            .unwrap();
2665        let mut disk = Fx::disk();
2666        disk.worktree_reclaim_bytes = 300;
2667        disk.min_free_bytes = u64::MAX;
2668        // Pressure must trigger reclamation even when aggregate usage is below cap.
2669        disk.worktree_limit_bytes = u64::MAX;
2670        let out = sweep(&f.home, &disk, Fx::now()).await;
2671        assert_eq!(out.freed_bytes, 1000, "{out:?}");
2672        assert!(!output.exists());
2673        assert!(
2674            small.candidates[0]
2675                .worktree
2676                .join("target/debug/output")
2677                .exists()
2678        );
2679    }
2680
2681    #[tokio::test]
2682    async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2683        let f = Fx::new().await;
2684        let first = f.run("bc00", true).await;
2685        let second = f.run("bc01", true).await;
2686        for state in [&first, &second] {
2687            Fx::target(&state.candidates[0].worktree);
2688        }
2689        // Non-declared Cargo data must not determine whether outputs fit.
2690        std::fs::create_dir_all(
2691            first.candidates[0]
2692                .worktree
2693                .join("target/debug/incremental"),
2694        )
2695        .unwrap();
2696        std::fs::write(
2697            first.candidates[0]
2698                .worktree
2699                .join("target/debug/incremental/cache"),
2700            [0; 1000],
2701        )
2702        .unwrap();
2703        let mut disk = Fx::disk();
2704        disk.worktree_reclaim_bytes = 0;
2705        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2706        disk.worktree_reclaim_bytes = 300;
2707        let out = sweep(&f.home, &disk, Fx::now()).await;
2708        assert_eq!(out.freed_bytes, 200, "{out:?}");
2709        assert!(
2710            !first.candidates[0]
2711                .worktree
2712                .join("target/debug/output")
2713                .exists()
2714        );
2715        assert!(
2716            second.candidates[0]
2717                .worktree
2718                .join("target/debug/output")
2719                .exists()
2720        );
2721        assert!(
2722            out.retained
2723                .iter()
2724                .any(|r| r.contains("declared outputs exceed"))
2725        );
2726    }
2727
2728    #[tokio::test]
2729    async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2730        let f = Fx::new().await;
2731        let mut protected = f.run("bd00", true).await;
2732        let later = f.run("bd01", true).await;
2733        for state in [&protected, &later] {
2734            Fx::target(&state.candidates[0].worktree);
2735        }
2736        let mut disk = Fx::disk();
2737        disk.worktree_reclaim_bytes = 100;
2738        disk.fold_grace_secs = 3600;
2739        let out = sweep(&f.home, &disk, Fx::now()).await;
2740        assert_eq!(out.freed_bytes, 0);
2741        assert!(
2742            out.retained
2743                .iter()
2744                .any(|r| r.contains("within grace period"))
2745        );
2746        disk.fold_grace_secs = 0;
2747        protected.driver_exited = false;
2748        protected.driver_pid = Some(std::process::id());
2749        protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2750        protected
2751            .save_under_at(&f.home, protected.updated_at)
2752            .unwrap();
2753        let out = sweep(&f.home, &disk, Fx::now()).await;
2754        assert!(
2755            out.retained
2756                .iter()
2757                .any(|r| r.contains("run live or ownership unknown"))
2758        );
2759        assert_eq!(
2760            out.freed_bytes, 200,
2761            "live target must not consume exception: {out:?}"
2762        );
2763        let output = protected.candidates[0].worktree.join("target/debug/output");
2764        assert!(output.exists());
2765        protected.driver_exited = true;
2766        protected.save_under(&f.home).unwrap();
2767        let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2768        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2769        assert!(output.exists());
2770        drop(owner);
2771        let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2772        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2773        assert!(output.exists());
2774        drop(cargo);
2775        let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2776        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2777        assert!(output.exists());
2778        drop(lease);
2779        assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2780        assert!(!output.exists());
2781    }
2782
2783    #[tokio::test]
2784    async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2785        for remove_root in [true, false] {
2786            let f = Fx::new().await;
2787            let stale = Fx::new().await;
2788            let mut missing = stale.run("aa00", false).await;
2789            Fx::target(&missing.candidates[0].worktree);
2790            missing.save_under(&f.home).unwrap();
2791            if remove_root {
2792                std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2793                // An existing non-repository proves the absent root is checked
2794                // before querying Git, even when no registrations are cached.
2795                std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2796            } else {
2797                std::fs::remove_dir_all(&stale.repo).unwrap();
2798            }
2799            let healthy = f.run("aa01", false).await;
2800            Fx::target(&healthy.candidates[0].worktree);
2801
2802            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2803            assert!(report.failures.is_empty(), "{report:?}");
2804            assert_eq!(report.freed_bytes, 200, "{report:?}");
2805            assert!(
2806                !healthy.candidates[0]
2807                    .worktree
2808                    .join("target/debug/output")
2809                    .exists()
2810            );
2811            if !remove_root {
2812                // Without the repository, ownership cannot be validated.
2813                assert!(
2814                    missing.candidates[0]
2815                        .worktree
2816                        .join("target/debug/output")
2817                        .exists()
2818                );
2819            }
2820        }
2821    }
2822
2823    #[tokio::test]
2824    async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2825        let f = Fx::new().await;
2826        let mut missing = f.run("aa00", false).await;
2827        Fx::target(&missing.candidates[0].worktree);
2828        missing.repo = missing.candidates[0].worktree.clone();
2829        missing.save_under(&f.home).unwrap();
2830        std::fs::remove_dir_all(&missing.repo).unwrap();
2831        assert!(missing.worktree_root().is_dir());
2832        assert!(f.repo.is_dir());
2833
2834        let healthy = f.run("aa01", false).await;
2835        Fx::target(&healthy.candidates[0].worktree);
2836
2837        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2838        assert!(report.failures.is_empty(), "{report:?}");
2839        assert_eq!(report.freed_bytes, 200, "{report:?}");
2840        assert!(
2841            !healthy.candidates[0]
2842                .worktree
2843                .join("target/debug/output")
2844                .exists()
2845        );
2846    }
2847
2848    #[tokio::test]
2849    async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2850        let f = Fx::new().await;
2851        let broken = Fx::new().await;
2852        let mut invalid = broken.run("aa00", false).await;
2853        Fx::target(&invalid.candidates[0].worktree);
2854        invalid.save_under(&f.home).unwrap();
2855        std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2856        let healthy = f.run("aa01", false).await;
2857        Fx::target(&healthy.candidates[0].worktree);
2858
2859        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2860        assert_eq!(report.failures.len(), 1, "{report:?}");
2861        assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2862        assert_eq!(report.freed_bytes, 200, "{report:?}");
2863        assert!(
2864            !healthy.candidates[0]
2865                .worktree
2866                .join("target/debug/output")
2867                .exists()
2868        );
2869        assert!(
2870            invalid.candidates[0]
2871                .worktree
2872                .join("target/debug/output")
2873                .exists()
2874        );
2875    }
2876
2877    async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2878        let path = reconcile::run(&f.home, &task.id, &source.id, None)
2879            .await
2880            .unwrap();
2881        serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2882    }
2883
2884    #[tokio::test]
2885    async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
2886        use crate::{
2887            chat_control::{Operation, dispatch},
2888            talk::{self, Talks},
2889        };
2890        let f = Fx::new().await;
2891        let source = f.run("ae00", true).await;
2892        let clean = f.run("ae01", false).await;
2893        let queue = crate::queue::Queue::at(f.home.join("queue"));
2894        let talks = Talks::at(f.home.join("talks"));
2895        let open_talk = |id: &str| -> talk::Talk {
2896            let mut conversation = serde_json::from_value(serde_json::json!({
2897                "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
2898                "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
2899                "seat":crate::agent::SeatState::new("talk","worker",1)
2900            }))
2901            .unwrap();
2902            talks.put(&mut conversation).unwrap();
2903            conversation
2904        };
2905        let conversation = open_talk("talk-owner");
2906        let other = open_talk("talk-other");
2907        // A task this chat did not file: the owner's request covers it too.
2908        let task = f.task(&[source.clone(), clean.clone()]);
2909        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2910        let operation = |apply: Option<&str>| Operation::ReconcileRetention {
2911            task: task.id.clone(),
2912            authority: source.id.clone(),
2913            apply: apply.map(str::to_owned),
2914        };
2915        // A plan saved outside the chat is not applied through it.
2916        let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
2917            .await
2918            .unwrap();
2919        let legacy: serde_json::Value =
2920            serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
2921        let legacy_id = legacy["id"].as_str().unwrap();
2922        assert!(
2923            dispatch(
2924                &queue,
2925                &f.home,
2926                &talks,
2927                &conversation.id,
2928                operation(Some(legacy_id))
2929            )
2930            .await
2931            .is_err()
2932        );
2933        let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
2934            .await
2935            .unwrap();
2936        assert!(summary.contains("Complete review summary"));
2937        assert!(summary.contains(&source.id) && summary.contains(&clean.id));
2938        let root = f.home.join("retention-reconciliations");
2939        let displayed: serde_json::Value =
2940            serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
2941        let id = displayed["plan"].as_str().unwrap();
2942        let dir = root.join(id);
2943        let plan: serde_json::Value =
2944            serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
2945        assert_eq!(plan["version"], 2);
2946        assert!(clean.candidates[0].worktree.exists());
2947        assert!(!dir.join("approved.json").exists());
2948        // The plan stays bound to the conversation it was saved for.
2949        assert!(
2950            dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
2951                .await
2952                .is_err()
2953        );
2954        assert!(!dir.join("approved.json").exists());
2955        // No owner phrase or quote: the request itself applies the plan. An
2956        // interruption after the durable fence keeps the audit for a retry.
2957        let error = reconcile::interrupt_chat_after_fence(
2958            &f.home,
2959            &task,
2960            id,
2961            reconcile::ChatRequest {
2962                talks: &talks,
2963                conversation: &conversation.id,
2964            },
2965        )
2966        .await
2967        .unwrap_err();
2968        assert!(
2969            error.to_string().contains("simulated interruption"),
2970            "{error:#}"
2971        );
2972        assert!(clean.candidates[0].worktree.exists());
2973        assert!(dir.join("approved.json").exists());
2974        dispatch(
2975            &queue,
2976            &f.home,
2977            &talks,
2978            &conversation.id,
2979            operation(Some(id)),
2980        )
2981        .await
2982        .unwrap();
2983        assert!(!clean.candidates[0].worktree.exists());
2984        assert!(source.candidates[0].worktree.exists());
2985        let audit: serde_json::Value =
2986            serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
2987        assert_eq!(audit["version"], 2);
2988        assert_eq!(
2989            audit["owner"],
2990            serde_json::json!({"conversation": conversation.id})
2991        );
2992        dispatch(
2993            &queue,
2994            &f.home,
2995            &talks,
2996            &conversation.id,
2997            operation(Some(id)),
2998        )
2999        .await
3000        .unwrap();
3001    }
3002
3003    #[tokio::test]
3004    async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3005        let f = Fx::new().await;
3006        let source = f.run("af00", true).await;
3007        let mut clean = f.run("af01", false).await;
3008        let mut dirty = f.run("af02", false).await;
3009        for state in [&mut clean, &mut dirty] {
3010            state.status = RunStatus::Prep;
3011            state.prep_complete = Some(false);
3012            state.candidates[0].empty = false;
3013            state.save_under(&f.home).unwrap();
3014        }
3015        std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3016        let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3017        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3018        let plan = reconciliation_plan(&f, &task, &source).await;
3019        let selected: Vec<_> = plan["candidates"]
3020            .as_array()
3021            .unwrap()
3022            .iter()
3023            .filter(|e| e["selected"] == true)
3024            .collect();
3025        assert_eq!(selected.len(), 1, "{plan:#}");
3026        assert_eq!(selected[0]["run"], clean.id);
3027        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3028            .await
3029            .unwrap();
3030        assert!(!clean.candidates[0].worktree.exists());
3031        assert!(dirty.candidates[0].worktree.exists());
3032        assert!(
3033            RunState::load_under(&dirty.id, &f.home)
3034                .unwrap()
3035                .retired_to
3036                .is_none()
3037        );
3038    }
3039
3040    #[tokio::test]
3041    async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3042        let f = Fx::new().await;
3043        let mut source = f.run("aa00", true).await;
3044        let alternative = f.run("aa01", true).await;
3045        let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3046            .await
3047            .unwrap();
3048        source.reviewed_commits = Some(vec![head.clone()]);
3049        source.save_under(&f.home).unwrap();
3050        let mut runs = vec![source.clone(), alternative.clone()];
3051        for i in 2..14 {
3052            runs.push(f.run(&format!("aa{i:02}"), false).await);
3053        }
3054        let mut task = f.task(&runs);
3055        task.pr_recovery = Some(crate::pr_recovery::Selection {
3056            url: "https://example.invalid/pull/1".into(),
3057            branch: "selected".into(),
3058            head: head.clone(),
3059            source_head: Some(head.clone()),
3060            source_run: Some(source.id.clone()),
3061            source_proof: None,
3062            disposition: Some(crate::pr_recovery::Disposition::Land),
3063            instruction: task.instruction.clone(),
3064            approved_at: Timestamp::now(),
3065        });
3066        source.pr_recovery = task.pr_recovery.clone();
3067        source.save_under(&f.home).unwrap();
3068        runs[0] = source.clone();
3069        crate::queue::Queue::at(f.home.join("queue"))
3070            .put(&mut task)
3071            .unwrap();
3072        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3073        let task_before = serde_json::to_value(&task).unwrap();
3074        let mut report = Report::default();
3075        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3076        assert_eq!(
3077            report.retired, 0,
3078            "automatic cleanup must still refuse blanket markers"
3079        );
3080        assert!(
3081            report
3082                .retained
3083                .iter()
3084                .any(|reason| reason.contains("explicitly retained"))
3085        );
3086        let mut total = 0;
3087        for expected in [5, 5, 2, 0] {
3088            let plan = reconciliation_plan(&f, &task, &source).await;
3089            let selected: Vec<_> = plan["candidates"]
3090                .as_array()
3091                .unwrap()
3092                .iter()
3093                .filter(|e| e["selected"] == true)
3094                .collect();
3095            assert_eq!(selected.len(), expected, "{plan:#}");
3096            for e in &selected {
3097                assert!(
3098                    RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3099                        .unwrap()
3100                        .retired_to
3101                        .is_none()
3102                );
3103            }
3104            let operation = plan["id"].as_str().unwrap();
3105            for _ in 0..2 {
3106                reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3107                    .await
3108                    .unwrap();
3109            }
3110            total += expected;
3111        }
3112        assert_eq!(total, 12);
3113        for run in &runs {
3114            let dir = f.home.join("runs").join(&run.id);
3115            assert!(dir.join("run.json").exists());
3116            assert!(dir.join("retain-recovery-sources").exists());
3117            assert!(
3118                git::branch_exists(&f.repo, &run.candidates[0].branch)
3119                    .await
3120                    .unwrap()
3121            );
3122        }
3123        assert!(source.candidates[0].worktree.exists());
3124        assert!(alternative.candidates[0].worktree.exists());
3125        assert_eq!(
3126            git::rev_parse(&f.repo, &source.candidates[0].branch)
3127                .await
3128                .unwrap(),
3129            head
3130        );
3131        assert_eq!(
3132            serde_json::to_value(
3133                crate::queue::Queue::at(f.home.join("queue"))
3134                    .get(&task.id)
3135                    .unwrap()
3136            )
3137            .unwrap(),
3138            task_before
3139        );
3140        assert_eq!(
3141            runs.iter()
3142                .filter(|r| RunState::load_under(&r.id, &f.home)
3143                    .unwrap()
3144                    .retired_to
3145                    .is_none())
3146                .count(),
3147            2
3148        );
3149    }
3150
3151    #[tokio::test]
3152    async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3153        let f = Fx::new().await;
3154        let source = f.run("ab00", true).await;
3155        let mut runs = vec![source.clone()];
3156        for i in 1..9 {
3157            runs.push(f.run(&format!("ab{i:02}"), false).await);
3158        }
3159        std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3160        std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3161        std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3162        std::fs::write(
3163            runs[3].candidates[0].worktree.join("scratch/source"),
3164            "ignored source",
3165        )
3166        .unwrap();
3167        std::fs::write(
3168            runs[4].worktree_root().join("unknown"),
3169            "unregistered source",
3170        )
3171        .unwrap();
3172        let checkpoint = git::git(
3173            &runs[5].candidates[0].worktree,
3174            &[
3175                "rev-parse",
3176                "--path-format=absolute",
3177                "--git-path",
3178                "sequencer",
3179            ],
3180        )
3181        .await
3182        .unwrap();
3183        std::fs::create_dir(checkpoint.trim()).unwrap();
3184        runs[6].driver_exited = false;
3185        runs[6].driver_pid = Some(std::process::id());
3186        runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3187        runs[6].save_under(&f.home).unwrap();
3188        runs[7].driver_exited = false;
3189        runs[7].driver_pid = None;
3190        runs[7].driver_claimed = None; // Legacy record without first-claim evidence.
3191        runs[7].save_under(&f.home).unwrap();
3192        assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3193
3194        let hidden = &runs[8].candidates[0].worktree;
3195        git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3196            .await
3197            .unwrap();
3198        std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3199        assert!(
3200            git::git(hidden, &["status", "--porcelain"])
3201                .await
3202                .unwrap()
3203                .is_empty()
3204        );
3205        let task = f.task(&runs);
3206        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3207        let plan = reconciliation_plan(&f, &task, &source).await;
3208        assert!(
3209            plan["candidates"]
3210                .as_array()
3211                .unwrap()
3212                .iter()
3213                .all(|e| e["selected"] == false),
3214            "{plan:#}"
3215        );
3216        for run in &runs {
3217            assert!(run.candidates[0].worktree.exists());
3218        }
3219    }
3220
3221    #[tokio::test]
3222    async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3223        let f = Fx::new().await;
3224        let source = f.run("ac00", true).await;
3225        let empty = f.run("ac01", false).await;
3226        let task = f.task(&[source.clone(), empty.clone()]);
3227        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3228        let plan = reconciliation_plan(&f, &task, &source).await;
3229        std::fs::write(
3230            f.home.join("runs").join(&source.id).join("new-report"),
3231            "new review checkpoint",
3232        )
3233        .unwrap();
3234        assert!(
3235            reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3236                .await
3237                .is_err()
3238        );
3239        assert!(empty.candidates[0].worktree.exists());
3240        assert!(
3241            RunState::load_under(&empty.id, &f.home)
3242                .unwrap()
3243                .retired_to
3244                .is_none()
3245        );
3246        let marker = f
3247            .home
3248            .join("runs")
3249            .join(&empty.id)
3250            .join("retain-recovery-sources");
3251        std::fs::write(&marker, "different owner").unwrap();
3252        let plan = reconciliation_plan(&f, &task, &source).await;
3253        assert!(
3254            plan["candidates"]
3255                .as_array()
3256                .unwrap()
3257                .iter()
3258                .all(|e| e["selected"] == false)
3259        );
3260    }
3261
3262    #[tokio::test]
3263    async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3264        for partial in [false, true] {
3265            let f = Fx::new().await;
3266            let source = f.run("ab10", true).await;
3267            let first = f.run("ab11", false).await;
3268            let second = f.run("ab12", false).await;
3269            let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3270            crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3271            let plan = reconciliation_plan(&f, &task, &source).await;
3272            let id = plan["id"].as_str().unwrap();
3273            let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3274            // Matching bytes without this operation's fence are not trusted.
3275            std::fs::write(&staging, b"").unwrap();
3276            assert!(
3277                reconcile::run(&f.home, &task.id, &source.id, Some(id))
3278                    .await
3279                    .is_err()
3280            );
3281            assert!(staging.exists());
3282            std::fs::remove_file(&staging).unwrap();
3283            let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3284                .await
3285                .unwrap_err();
3286            assert!(
3287                error.to_string().contains("before run state rename"),
3288                "{error:#}"
3289            );
3290            assert!(
3291                RunState::load_under(&first.id, &f.home)
3292                    .unwrap()
3293                    .retired_to
3294                    .is_some()
3295            );
3296            assert!(
3297                RunState::load_under(&second.id, &f.home)
3298                    .unwrap()
3299                    .retired_to
3300                    .is_none()
3301            );
3302            assert!(first.candidates[0].worktree.exists());
3303            assert!(second.candidates[0].worktree.exists());
3304            let interrupted = std::fs::read(&staging).unwrap();
3305            // An unrelated file must survive a refused retry unchanged.
3306            std::fs::write(&staging, b"unrelated recovery source").unwrap();
3307            assert!(
3308                reconcile::run(&f.home, &task.id, &source.id, Some(id))
3309                    .await
3310                    .is_err()
3311            );
3312            assert_eq!(
3313                std::fs::read(&staging).unwrap(),
3314                b"unrelated recovery source"
3315            );
3316            std::fs::write(&staging, interrupted).unwrap();
3317            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3318                .await
3319                .unwrap();
3320            assert!(!staging.exists());
3321            assert!(!first.candidates[0].worktree.exists());
3322            assert!(!second.candidates[0].worktree.exists());
3323            assert_eq!(
3324                RunState::load_under(&second.id, &f.home)
3325                    .unwrap()
3326                    .retired_to,
3327                Some(source.id)
3328            );
3329        }
3330    }
3331
3332    #[tokio::test]
3333    async fn reconciliation_resumes_only_its_exact_durable_fence() {
3334        let f = Fx::new().await;
3335        let source = f.run("ad00", true).await;
3336        let empty = f.run("ad01", false).await;
3337        let task = f.task(&[source.clone(), empty.clone()]);
3338        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3339        let plan = reconciliation_plan(&f, &task, &source).await;
3340        let id = plan["id"].as_str().unwrap();
3341        assert!(
3342            reconcile::interrupt_after_fence(&f.home, &task, id)
3343                .await
3344                .is_err()
3345        );
3346        assert!(empty.candidates[0].worktree.exists());
3347        assert_eq!(
3348            RunState::load_under(&empty.id, &f.home)
3349                .unwrap()
3350                .retired_to
3351                .as_ref(),
3352            Some(&source.id)
3353        );
3354        reconcile::run(&f.home, &task.id, &source.id, Some(id))
3355            .await
3356            .unwrap();
3357        assert!(!empty.candidates[0].worktree.exists());
3358        // Model an interrupted removal: restore precisely the approved base worktree.
3359        git::worktree_add_branch(
3360            &f.repo,
3361            &empty.candidates[0].worktree,
3362            "restore-empty",
3363            &f.base,
3364        )
3365        .await
3366        .unwrap();
3367        // Recreated Git administration differs from approval and is refused.
3368        assert!(
3369            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3370                .await
3371                .is_err()
3372        );
3373        assert!(empty.candidates[0].worktree.exists());
3374        let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3375        changed.prep_failure = Some("new owner edit".into());
3376        changed.save_under(&f.home).unwrap();
3377        assert!(
3378            reconcile::run(&f.home, &task.id, &source.id, Some(id))
3379                .await
3380                .is_err()
3381        );
3382    }
3383
3384    #[tokio::test]
3385    async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3386        let f = Fx::new().await;
3387        let source = f.run("ae00", true).await;
3388        let mut prep = f.run("ae01", false).await;
3389        let candidate = prep.candidates.pop().unwrap();
3390        prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3391        prep.save_under(&f.home).unwrap();
3392        let reset = f.run("ae02", false).await;
3393        let wt = &reset.candidates[0].worktree;
3394        std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3395        git::git(wt, &["add", "."]).await.unwrap();
3396        git::git(wt, &["commit", "-m", "unique implementation"])
3397            .await
3398            .unwrap();
3399        git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3400        let unknown = f.run("ae03", false).await;
3401        let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3402        let mut raw: serde_json::Value =
3403            serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3404        raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3405        std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3406        let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3407        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3408        let plan = reconciliation_plan(&f, &task, &source).await;
3409        let selected: Vec<_> = plan["candidates"]
3410            .as_array()
3411            .unwrap()
3412            .iter()
3413            .filter(|e| e["selected"] == true)
3414            .collect();
3415        assert_eq!(selected.len(), 1, "{plan:#}");
3416        assert_eq!(selected[0]["run"], prep.id);
3417        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3418            .await
3419            .unwrap();
3420        assert!(!candidate.worktree.exists());
3421        assert!(reset.candidates[0].worktree.exists());
3422        assert!(unknown.candidates[0].worktree.exists());
3423    }
3424
3425    #[tokio::test]
3426    async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3427        let f = Fx::new().await;
3428        let mut state = f.run("aaaa", true).await;
3429        let candidate = state.candidates[0].worktree.clone();
3430        std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3431        Fx::target(&candidate);
3432        for seat in ["review-1", "review-2"] {
3433            let wt = state.worktree_root().join(seat);
3434            git::worktree_add_detached(&f.repo, &wt, &f.base)
3435                .await
3436                .unwrap();
3437            Fx::target(&wt);
3438        }
3439        state.status = RunStatus::Landing;
3440        state.save_under(&f.home).unwrap();
3441        let second = Fx::new().await;
3442        let other = second.run("bbbb", true).await;
3443        // A global store may contain repositories with different worktree roots.
3444        let mut imported = other.clone();
3445        imported.save_under(&f.home).unwrap();
3446        Fx::target(&other.candidates[0].worktree);
3447        let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3448        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3449        assert!(
3450            candidate.join("target/debug/output").exists(),
3451            "active writer protected"
3452        );
3453        assert!(
3454            !state
3455                .worktree_root()
3456                .join("review-1/target/debug/output")
3457                .exists()
3458        );
3459        assert!(
3460            !state
3461                .worktree_root()
3462                .join("review-2/target/debug/output")
3463                .exists()
3464        );
3465        assert!(
3466            !other.candidates[0]
3467                .worktree
3468                .join("target/debug/output")
3469                .exists()
3470        );
3471        assert!(out.freed_bytes >= 600);
3472        drop(guard);
3473        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3474        assert!(out.freed_bytes >= 200);
3475        assert_eq!(
3476            std::fs::read_to_string(candidate.join("source")).unwrap(),
3477            "uncommitted work"
3478        );
3479        assert!(
3480            git::branch_exists(&f.repo, &state.candidates[0].branch)
3481                .await
3482                .unwrap()
3483        );
3484    }
3485
3486    #[tokio::test]
3487    async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3488        let f = Fx::new().await;
3489        let state = f.run("cc01", false).await;
3490        let wt = &state.candidates[0].worktree;
3491        Fx::target(wt);
3492        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3493            let dir = wt.join("target").join(dir);
3494            std::fs::create_dir_all(&dir).unwrap();
3495            std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3496        }
3497        // A dep-info dependency is an input, never deletion authority.
3498        let manifest = wt.join("target/debug/output.d");
3499        let output = wt.join("target/debug/output");
3500        std::fs::write(
3501            &manifest,
3502            format!(
3503                "{}: {}\n",
3504                output.display(),
3505                wt.join("target/debug/notes.txt").display()
3506            ),
3507        )
3508        .unwrap();
3509        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3510        assert_eq!(report.freed_bytes, 200);
3511        assert!(!output.exists());
3512        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3513            assert_eq!(
3514                std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3515                "user evidence"
3516            );
3517        }
3518        // Metadata remains available for inspection and future safe rebuilds.
3519        assert!(manifest.exists());
3520        assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3521        std::fs::write(&output, "user note replacing an old binary").unwrap();
3522        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3523        assert_eq!(report.freed_bytes, 0);
3524        assert_eq!(
3525            std::fs::read_to_string(&output).unwrap(),
3526            "user note replacing an old binary"
3527        );
3528    }
3529
3530    #[tokio::test]
3531    async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3532        let f = Fx::new().await;
3533        let state = f.run("cc02", false).await;
3534        let wt = &state.candidates[0].worktree;
3535        Fx::target(wt);
3536        let mut disk = Fx::disk();
3537        disk.worktree_limit_bytes = u64::MAX;
3538        let now = Fx::now();
3539        let first = sweep(&f.home, &disk, now).await;
3540        std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3541        assert_eq!(
3542            sweep(&f.home, &disk, now).await.measured_bytes,
3543            first.measured_bytes
3544        );
3545        assert!(
3546            sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3547                .await
3548                .measured_bytes
3549                > first.measured_bytes
3550        );
3551        disk.worktree_limit_bytes = 0;
3552        assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3553    }
3554
3555    #[tokio::test]
3556    async fn unfinished_source_fold_reports_its_retention_reason() {
3557        let f = Fx::new().await;
3558        let mut source = f.run("cc03", true).await;
3559        let task = f.task(&[source.clone()]);
3560        let error = crate::graph::fold_run(&mut source, true, &f.home)
3561            .await
3562            .unwrap_err();
3563        assert!(error.to_string().contains(&task.id));
3564        assert!(source.candidates[0].worktree.exists());
3565        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3566        let error = crate::graph::fold_run(&mut source, true, &f.home)
3567            .await
3568            .unwrap_err();
3569        assert!(error.to_string().contains("explicitly retained"));
3570    }
3571
3572    #[tokio::test]
3573    async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3574        let f = Fx::new().await;
3575        let mut source = f.run("cc04", true).await;
3576        let mut task = f.task(&[source.clone()]);
3577        task.hold_manual(Some("queue settlement pending".into()));
3578        let queue = crate::queue::Queue::at(f.home.join("queue"));
3579        queue.put(&mut task).unwrap();
3580        source.merge = Some(crate::run::MergeOutcome {
3581            mode: crate::config::MergeMode::Pr,
3582            ok: true,
3583            detail: "confirmed merged".into(),
3584            empty: false,
3585        });
3586        // Publication success and terminal labels alone are insufficient.
3587        for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3588            source.status = status;
3589            assert!(
3590                crate::graph::fold_run(&mut source, true, &f.home)
3591                    .await
3592                    .is_err()
3593            );
3594            assert!(source.candidates[0].worktree.exists());
3595        }
3596        git::git(
3597            &f.repo,
3598            &["merge", "--ff-only", &source.candidates[0].branch],
3599        )
3600        .await
3601        .unwrap();
3602        git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3603            .await
3604            .unwrap();
3605        source.status = RunStatus::Merged;
3606        source.save_under(&f.home).unwrap();
3607        assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3608        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3609        assert!(
3610            crate::graph::fold_run(&mut source, true, &f.home)
3611                .await
3612                .unwrap_err()
3613                .to_string()
3614                .contains("explicitly retained")
3615        );
3616        // The fixture's explicit marker is removed only to exercise ordinary
3617        // confirmed-merge cleanup; no production cleanup removes such markers.
3618        std::fs::remove_file(
3619            f.home
3620                .join("runs")
3621                .join(&source.id)
3622                .join("retain-recovery-sources"),
3623        )
3624        .unwrap();
3625        let (folded, unreadable) = crate::clean::fold_due(
3626            &f.home.join("runs"),
3627            &f.home,
3628            &f.bay,
3629            &Fx::disk(),
3630            Fx::now(),
3631        )
3632        .await
3633        .unwrap();
3634        assert_eq!((folded, unreadable), (1, 0));
3635        assert!(!source.candidates[0].worktree.exists());
3636        let evidence = f
3637            .home
3638            .join("runs")
3639            .join(&source.id)
3640            .join("artifacts/fold-evidence");
3641        let names: Vec<_> = std::fs::read_dir(evidence)
3642            .unwrap()
3643            .map(|entry| entry.unwrap().file_name())
3644            .collect();
3645        assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3646        assert_eq!(
3647            queue.get(&task.id).unwrap().status,
3648            crate::queue::TaskStatus::Held
3649        );
3650    }
3651
3652    #[tokio::test]
3653    async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3654        for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3655            let f = Fx::new().await;
3656            let mut state = f.run("cc06", true).await;
3657            let candidate = state.candidates[0].clone();
3658            let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3659            std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3660            git::rescue_commit(&candidate.worktree, "second candidate change")
3661                .await
3662                .unwrap();
3663            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3664            state.tally = Some(crate::run::Tally {
3665                first_choice: Default::default(),
3666                borda: Default::default(),
3667                winner: 'A',
3668                rankings: 0,
3669                unanimous_initial: true,
3670                deliberated: false,
3671                changed_votes: 0,
3672                unanimous_final: true,
3673                tie_break: None,
3674                judges: 0,
3675                present: 0,
3676                quorum: 0,
3677                met_quorum: true,
3678                uncontested: Some("fixture".into()),
3679            });
3680            // A reviewer pinned to an intermediate candidate commit must also
3681            // be reclaimable, although neither commit ID lands in the squash.
3682            let reviewer = state.worktree_root().join("review-1");
3683            git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3684                .await
3685                .unwrap();
3686            // Concurrent base changes and later base advances prevent a mere
3687            // equality check against the current base tree from passing.
3688            std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3689            git::rescue_commit(&f.repo, "advance before squash")
3690                .await
3691                .unwrap();
3692            let published = if variant == "partial" {
3693                &intermediate
3694            } else {
3695                &head
3696            };
3697            git::git(&f.repo, &["merge", "--squash", published])
3698                .await
3699                .unwrap();
3700            git::git(&f.repo, &["commit", "-m", "squash candidate"])
3701                .await
3702                .unwrap();
3703            let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3704            assert!(
3705                !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3706                    .await
3707                    .unwrap()
3708                    .ok()
3709            );
3710            std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3711            git::rescue_commit(&f.repo, "advance after squash")
3712                .await
3713                .unwrap();
3714            git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3715                .await
3716                .unwrap();
3717            if matches!(variant, "checkpoint" | "reviewer") {
3718                // A reset hides unique work from HEAD, but not its reflog.
3719                std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3720                git::rescue_commit(&reviewer, "unpublished checkpoint")
3721                    .await
3722                    .unwrap();
3723                if variant == "checkpoint" {
3724                    git::git(&reviewer, &["reset", "--hard", &intermediate])
3725                        .await
3726                        .unwrap();
3727                }
3728            }
3729            if variant == "loser" {
3730                let mut loser = candidate.clone();
3731                loser.label = 'B';
3732                loser.branch = "magi/cc06/B".into();
3733                loser.worktree = state.worktree_root().join("cand-B");
3734                git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3735                    .await
3736                    .unwrap();
3737                std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3738                git::rescue_commit(&loser.worktree, "independent loser")
3739                    .await
3740                    .unwrap();
3741                state.candidates.push(loser);
3742            }
3743            state.status = RunStatus::Merged;
3744            state.merge = Some(crate::run::MergeOutcome {
3745                mode: crate::config::MergeMode::Pr,
3746                ok: true,
3747                detail: "candidate A squash merged".into(),
3748                empty: false,
3749            });
3750            state.save_under(&f.home).unwrap();
3751            let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3752            let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3753            result.unwrap();
3754            assert!(
3755                !candidate.worktree.exists() && !reviewer.exists(),
3756                "{variant}"
3757            );
3758            assert!(
3759                !git::branch_exists(&f.repo, &candidate.branch)
3760                    .await
3761                    .unwrap()
3762            );
3763            let artifacts = f
3764                .home
3765                .join("runs")
3766                .join(&state.id)
3767                .join("artifacts/fold-evidence");
3768            assert_eq!(
3769                artifacts.join(format!("{head}.patch")).exists(),
3770                variant == "partial",
3771                "only a partially published winner needs evidence: {variant}"
3772            );
3773            assert_eq!(
3774                artifacts.join(format!("{intermediate}.patch")).exists(),
3775                variant == "partial",
3776                "squash proof also covers intermediate reviewer tips: {variant}"
3777            );
3778            let patches: Vec<_> = std::fs::read_dir(&artifacts)
3779                .unwrap()
3780                .map(|entry| entry.unwrap().path())
3781                .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3782                .collect();
3783            let expected = match variant {
3784                "partial" => 2,
3785                "loser" | "reviewer" => 1,
3786                _ => 0,
3787            };
3788            assert_eq!(patches.len(), expected, "{variant}");
3789            if variant == "loser" {
3790                assert!(
3791                    std::fs::read_to_string(&patches[0])
3792                        .unwrap()
3793                        .contains("losing candidate")
3794                );
3795            }
3796            assert!(artifacts.join("manifest.json").exists());
3797            if variant == "reviewer" {
3798                assert!(
3799                    std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3800                        .unwrap()
3801                        .contains("reviewer checkpoint")
3802                );
3803            }
3804            crate::graph::fold_run(&mut state, true, &f.home)
3805                .await
3806                .unwrap();
3807        }
3808    }
3809
3810    #[tokio::test]
3811    async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3812        for missing in [true, false] {
3813            let f = Fx::new().await;
3814            let mut state = f.run("cc05", true).await;
3815            // A fetched base exists, but these local review changes were not
3816            // incorporated by ancestry, so their recovery evidence is required.
3817            git::git(
3818                &f.repo,
3819                &["update-ref", "refs/remotes/origin/main", &f.base],
3820            )
3821            .await
3822            .unwrap();
3823            let candidate = state.candidates[0].clone();
3824            if !missing {
3825                std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3826                git::rescue_commit(&f.repo, "advance base before rebase")
3827                    .await
3828                    .unwrap();
3829                git::git(&candidate.worktree, &["rebase", "main"])
3830                    .await
3831                    .unwrap();
3832                // Match the runner's post-rebase synchronization, which
3833                // clears the merge backend's transient AUTO_MERGE checkpoint.
3834                git::sync_to_head(&candidate.worktree).await.unwrap();
3835            }
3836            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3837            // Model review-only state: no implementation-time snapshot exists.
3838            state.reviewed_commits = Some(Vec::new());
3839            state.candidates[0].worktree = state.worktree_root().join("under-review");
3840            git::git(
3841                &f.repo,
3842                &[
3843                    "worktree",
3844                    "move",
3845                    candidate.worktree.to_str().unwrap(),
3846                    state.candidates[0].worktree.to_str().unwrap(),
3847                ],
3848            )
3849            .await
3850            .unwrap();
3851            let reviewer = state.worktree_root().join("review-1");
3852            git::worktree_add_detached(&f.repo, &reviewer, &head)
3853                .await
3854                .unwrap();
3855            // A reset/rebase can leave unpublished recovery history. It must
3856            // not keep any remaining owned worktree alive indefinitely.
3857            std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3858            git::rescue_commit(&reviewer, "old review history")
3859                .await
3860                .unwrap();
3861            git::git(&reviewer, &["reset", "--hard", &head])
3862                .await
3863                .unwrap();
3864            if missing {
3865                std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3866            }
3867            let run_dir = f.home.join("runs").join(&state.id);
3868            let artifacts = run_dir.join("artifacts");
3869            std::fs::create_dir_all(&artifacts).unwrap();
3870            std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3871            if missing {
3872                std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3873            }
3874            state.status = RunStatus::Merged;
3875            state.merge = Some(crate::run::MergeOutcome {
3876                mode: crate::config::MergeMode::Pr,
3877                ok: true,
3878                detail: "confirmed squash merge".into(),
3879                empty: false,
3880            });
3881            state.save_under(&f.home).unwrap();
3882            // Unexpected external paths remain protected, even if absent.
3883            let owned = state.candidates[0].worktree.clone();
3884            state.candidates[0].worktree = f.home.join("missing-external");
3885            assert!(
3886                crate::graph::fold_run(&mut state, true, &f.home)
3887                    .await
3888                    .unwrap_err()
3889                    .to_string()
3890                    .contains("outside the inventoried")
3891            );
3892            state.candidates[0].worktree = owned;
3893            // A live output borrower blocks the entire preflight.
3894            let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
3895            assert!(
3896                crate::graph::fold_run(&mut state, true, &f.home)
3897                    .await
3898                    .is_err()
3899            );
3900            assert!(reviewer.exists());
3901            drop(guard);
3902            // Failure to persist evidence must not cause partial removal.
3903            let evidence = artifacts.join("fold-evidence");
3904            std::fs::write(&evidence, "blocked").unwrap();
3905            assert!(
3906                crate::graph::fold_run(&mut state, true, &f.home)
3907                    .await
3908                    .is_err()
3909            );
3910            assert!(reviewer.exists());
3911            std::fs::remove_file(&evidence).unwrap();
3912            std::fs::create_dir_all(&evidence).unwrap();
3913            let oversized = evidence.join("capacity-fixture");
3914            std::fs::File::create(&oversized)
3915                .unwrap()
3916                .set_len(64 * 1024 * 1024)
3917                .unwrap();
3918            assert!(
3919                crate::graph::fold_run(&mut state, true, &f.home)
3920                    .await
3921                    .is_err()
3922            );
3923            assert!(reviewer.exists());
3924            std::fs::remove_file(oversized).unwrap();
3925            // Ignored recovery files and unfinished Git operations still hold
3926            // all sources, even for a confirmed merge.
3927            std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
3928            std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
3929            assert!(
3930                crate::graph::fold_run(&mut state, true, &f.home)
3931                    .await
3932                    .is_err()
3933            );
3934            std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
3935            let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
3936                .await
3937                .unwrap();
3938            std::fs::write(&checkpoint, &head).unwrap();
3939            assert!(
3940                crate::graph::fold_run(&mut state, true, &f.home)
3941                    .await
3942                    .is_err()
3943            );
3944            std::fs::remove_file(checkpoint).unwrap();
3945            let (folded, unreadable) = crate::clean::fold_due(
3946                &f.home.join("runs"),
3947                &f.home,
3948                &f.bay,
3949                &Fx::disk(),
3950                Fx::now(),
3951            )
3952            .await
3953            .unwrap();
3954            if folded != 1 {
3955                panic!(
3956                    "normal cleanup failed: {:#}",
3957                    crate::graph::fold_run(&mut state, true, &f.home)
3958                        .await
3959                        .unwrap_err()
3960                );
3961            }
3962            assert_eq!((folded, unreadable), (1, 0));
3963            assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
3964            state = RunState::load_under(&state.id, &f.home).unwrap();
3965            assert!(state.candidates[0].folded);
3966            let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
3967            crate::graph::fold_run(&mut state, true, &f.home)
3968                .await
3969                .unwrap();
3970            assert_eq!(
3971                std::fs::read(evidence.join("manifest.json")).unwrap(),
3972                manifest
3973            );
3974            git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
3975                .await
3976                .unwrap();
3977            git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
3978            assert!(
3979                !git::git_raw(&f.repo, &["cat-file", "-e", &head])
3980                    .await
3981                    .unwrap()
3982                    .ok()
3983            );
3984            // Restore the modified file into a fresh repository from artifact
3985            // bytes alone; no original commit or blob is consulted.
3986            let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
3987            let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
3988                .as_str()
3989                .unwrap();
3990            let recovered = f.home.join("recovered");
3991            std::fs::create_dir_all(&recovered).unwrap();
3992            git::git(&recovered, &["init"]).await.unwrap();
3993            std::fs::write(
3994                recovered.join("source"),
3995                std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
3996            )
3997            .unwrap();
3998            git::git(
3999                &recovered,
4000                &[
4001                    "apply",
4002                    evidence.join(format!("{head}.patch")).to_str().unwrap(),
4003                ],
4004            )
4005            .await
4006            .unwrap();
4007            assert_eq!(
4008                std::fs::read_to_string(recovered.join("source")).unwrap(),
4009                "cc05"
4010            );
4011            assert!(
4012                !std::fs::read(evidence.join(format!("{head}.patch")))
4013                    .unwrap()
4014                    .is_empty()
4015            );
4016            assert_eq!(
4017                std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4018                "review findings"
4019            );
4020            if missing {
4021                assert_eq!(
4022                    std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4023                    "original snapshot"
4024                );
4025            } else {
4026                assert!(!artifacts.join("cand-A.patch").exists());
4027            }
4028            assert!(run_dir.join("run.json").exists());
4029        }
4030    }
4031
4032    #[tokio::test]
4033    async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4034        use std::os::unix::fs::PermissionsExt as _;
4035        let f = Fx::new().await;
4036        let source = f.run("dd03", true).await;
4037        let empty = f.run("dd04", false).await;
4038        let task = f.task(&[source, empty.clone()]);
4039        let entered = f.repo.join("proof-entered");
4040        let release = f.repo.join("proof-release");
4041        let hook = f.repo.join("proof-monitor");
4042        std::fs::write(&hook, format!(
4043            "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4044            entered.display(), release.display())).unwrap();
4045        std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4046        git::git(
4047            &f.repo,
4048            &["config", "core.fsmonitor", hook.to_str().unwrap()],
4049        )
4050        .await
4051        .unwrap();
4052        struct ReleaseProof(PathBuf);
4053        impl Drop for ReleaseProof {
4054            fn drop(&mut self) {
4055                let _ = std::fs::write(&self.0, "continue");
4056            }
4057        }
4058        let _release_on_panic = ReleaseProof(release.clone());
4059        let home = f.home.clone();
4060        let sweep = tokio::spawn(async move {
4061            let mut report = Report::default();
4062            retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4063            report
4064        });
4065        // These are deadlock watchdogs, not speed assertions. The proof stays
4066        // blocked until the task update completes, even on a loaded runner.
4067        let watchdog = std::time::Duration::from_secs(60);
4068        tokio::time::timeout(watchdog, async {
4069            while !entered.exists() {
4070                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4071            }
4072        })
4073        .await
4074        .expect("Git retirement proof did not enter the blocking hook");
4075        let queue = crate::queue::Queue::at(f.home.join("queue"));
4076        let mut selected = queue.get(&task.id).unwrap();
4077        selected.resume_override = Some(crate::queue::OperatorResume {
4078            question_id: "during-slow-proof".into(),
4079            at: Fx::now(),
4080            conductor_rehold: None,
4081            forced: false,
4082            pinned_run: Some(empty.id.clone()),
4083        });
4084        let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4085        // The proof is released only after the update returns, so correctness
4086        // is decided by ordering, not speed. The 60-second bound only detects
4087        // a writer blocked behind the proof; it is not a performance threshold.
4088        let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4089            Ok(result) => {
4090                result.unwrap().unwrap();
4091                true
4092            }
4093            Err(_) => {
4094                std::fs::write(&release, "continue").unwrap();
4095                update.await.unwrap().unwrap();
4096                false
4097            }
4098        };
4099        std::fs::write(release, "continue").unwrap();
4100        let report = tokio::time::timeout(watchdog, sweep)
4101            .await
4102            .expect("Git retirement proof did not finish after release")
4103            .unwrap();
4104        assert!(
4105            updated_while_blocked,
4106            "task writers must not wait behind Git retirement proof"
4107        );
4108        assert_eq!(report.retired, 0);
4109        assert!(report.retained.iter().any(|r| r.contains("task changed")));
4110        assert!(empty.candidates[0].worktree.exists());
4111        assert!(
4112            RunState::load_under(&empty.id, &f.home)
4113                .unwrap()
4114                .retired_to
4115                .is_none()
4116        );
4117    }
4118
4119    #[tokio::test]
4120    async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4121        let f = Fx::new().await;
4122        let source = f.run("dd01", true).await;
4123        let mut empty = f.run("dd02", false).await;
4124        let expected = f.task(&[source.clone(), empty.clone()]);
4125        let queue = crate::queue::Queue::at(f.home.join("queue"));
4126        let _claim = queue.claim(&expected.id).unwrap();
4127        // The proof holds run/output leases, never the task write lock.
4128        let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4129        let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4130        let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4131        let mut selected = queue.get(&expected.id).unwrap();
4132        selected.resume_override = Some(crate::queue::OperatorResume {
4133            question_id: "new-owner-decision".into(),
4134            at: Fx::now(),
4135            conductor_rehold: None,
4136            forced: false,
4137            pinned_run: Some(empty.id.clone()),
4138        });
4139        queue.put(&mut selected).unwrap();
4140        let error = commit_retirement(
4141            &f.home,
4142            &queue,
4143            &expected,
4144            &mut empty,
4145            &source.id,
4146            &expected.runs,
4147            Fx::now(),
4148        )
4149        .unwrap_err();
4150        assert!(error.to_string().contains("task changed"));
4151        assert!(
4152            RunState::load_under(&empty.id, &f.home)
4153                .unwrap()
4154                .retired_to
4155                .is_none()
4156        );
4157        assert!(empty.candidates[0].worktree.exists());
4158        assert_eq!(
4159            queue
4160                .get(&expected.id)
4161                .unwrap()
4162                .resume_override
4163                .unwrap()
4164                .pinned_run,
4165            Some(empty.id.clone())
4166        );
4167        assert!(
4168            !f.home
4169                .join("recovery-lineages")
4170                .join(format!("{}.json", expected.id))
4171                .exists()
4172        );
4173        // Re-reading the new snapshot must not bypass its explicit pin either.
4174        assert!(
4175            commit_retirement(
4176                &f.home,
4177                &queue,
4178                &selected,
4179                &mut empty,
4180                &source.id,
4181                &selected.runs,
4182                Fx::now()
4183            )
4184            .unwrap_err()
4185            .to_string()
4186            .contains("selected this recovery source")
4187        );
4188    }
4189
4190    #[tokio::test]
4191    async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4192        let f = Fx::new().await;
4193        let state = f.run("cccc", false).await;
4194        let wt = &state.candidates[0].worktree;
4195        Fx::target(wt);
4196        std::fs::write(wt.join("target/notes"), "user file").unwrap();
4197        std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4198        assert!(
4199            !declared_outputs(&wt.join("target"))
4200                .unwrap()
4201                .contains(&wt.join("target/debug/link"))
4202        );
4203        std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4204        let cargo = cargo_locks(&wt.join("target")).unwrap();
4205        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4206        assert_eq!(out.freed_bytes, 0);
4207        assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4208        drop(cargo);
4209        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4210        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4211        assert_eq!(out.freed_bytes, 0);
4212        drop(driver);
4213        let exclusive = OutputLease::take(wt, true).unwrap();
4214        assert!(
4215            OutputLease::borrow(wt).is_err(),
4216            "resume cannot start a writer during cleanup"
4217        );
4218        drop(exclusive);
4219        assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4220        assert_eq!(
4221            std::fs::read(wt.join("target/notes")).unwrap(),
4222            b"user file"
4223        );
4224    }
4225
4226    #[tokio::test]
4227    async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4228        for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4229            let f = Fx::new().await;
4230            let source = f.run("aa10", true).await;
4231            let empty = f.run("aa11", false).await;
4232            f.task(&[source, empty.clone()]);
4233            let wt = &empty.candidates[0].worktree;
4234            Fx::target(wt);
4235            let path = wt.join("target").join(leftover);
4236            if leftover.ends_with(".log") {
4237                std::fs::write(&path, "keep log\n").unwrap();
4238            } else {
4239                // Empty unknown directories are not present in residue_files.
4240                std::fs::create_dir(&path).unwrap();
4241            }
4242            let mut disk = Fx::disk();
4243            disk.auto_fold = true;
4244            let report = sweep(&f.home, &disk, Fx::now()).await;
4245            assert_eq!(report.freed_bytes, 200, "{report:?}");
4246            assert_eq!(report.retired, 0, "{report:?}");
4247            assert!(!wt.join("target/debug/output").exists());
4248            assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4249            assert!(path.exists());
4250            if leftover.ends_with(".log") {
4251                assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4252            }
4253            assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4254            assert!(worktrees(&empty).await.unwrap().contains(wt));
4255            assert!(
4256                git::branch_exists(&f.repo, &empty.candidates[0].branch)
4257                    .await
4258                    .unwrap()
4259            );
4260            assert!(
4261                RunState::load_under(&empty.id, &f.home)
4262                    .unwrap()
4263                    .retired_to
4264                    .is_none()
4265            );
4266        }
4267    }
4268
4269    #[tokio::test]
4270    async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4271        let f = Fx::new().await;
4272        let state = f.run("aa12", false).await;
4273        let wt = &state.candidates[0].worktree;
4274        Fx::target(wt);
4275        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4276        assert_eq!(report.freed_bytes, 200, "{report:?}");
4277        assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4278        for name in ["tmp", "magi-tools", "log"] {
4279            let path = wt.join("target").join(name);
4280            std::fs::create_dir(&path).unwrap();
4281            assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4282            std::fs::remove_dir(path).unwrap();
4283        }
4284        assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4285    }
4286
4287    #[tokio::test]
4288    async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4289        for invalid in [
4290            "tag",
4291            "identity",
4292            "tracked",
4293            "profile-file",
4294            "profile-symlink",
4295            "identity-symlink",
4296        ] {
4297            let f = Fx::new().await;
4298            let state = f.run("aa13", false).await;
4299            let wt = &state.candidates[0].worktree;
4300            Fx::target(wt);
4301            std::fs::create_dir(wt.join("target/tmp")).unwrap();
4302            let target = wt.join("target");
4303            match invalid {
4304                "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4305                "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4306                "tracked" => {
4307                    git::git(wt, &["add", "-f", "target/debug/output"])
4308                        .await
4309                        .unwrap();
4310                }
4311                "profile-file" => {
4312                    std::fs::write(target.join("release"), "keep").unwrap();
4313                }
4314                "profile-symlink" => {
4315                    std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4316                        .unwrap();
4317                }
4318                "identity-symlink" => {
4319                    std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4320                        .unwrap();
4321                    std::os::unix::fs::symlink(
4322                        target.join("saved-info"),
4323                        target.join(".rustc_info.json"),
4324                    )
4325                    .unwrap();
4326                }
4327                _ => unreachable!(),
4328            }
4329            let before = std::fs::read(target.join("debug/output")).unwrap();
4330            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4331            assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4332            assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4333            assert!(target.join("tmp").is_dir());
4334        }
4335    }
4336
4337    #[tokio::test]
4338    async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4339        let f = Fx::new().await;
4340        let source = f.run("0000", true).await;
4341        let empty = f.run("0001", false).await;
4342        let changed = f.run("0002", false).await;
4343        let unknown = f.run("0003", false).await;
4344        let unproven = f.run("0004", false).await;
4345        f.task(&[
4346            source.clone(),
4347            empty.clone(),
4348            changed.clone(),
4349            unknown.clone(),
4350            unproven.clone(),
4351        ]);
4352        for run in [&empty, &changed, &unknown] {
4353            Fx::target(&run.candidates[0].worktree);
4354        }
4355        let reviewer = empty.worktree_root().join("review-1");
4356        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4357            .await
4358            .unwrap();
4359        Fx::target(&reviewer);
4360        let mut disk = Fx::disk();
4361        disk.auto_fold = false;
4362        let report = sweep(&f.home, &disk, Fx::now()).await;
4363        assert_eq!(report.freed_bytes, 800, "{report:?}");
4364        let wt = &empty.candidates[0].worktree;
4365        assert!(wt.join("target/debug/output.d").exists());
4366        assert!(!wt.join("target/debug/output").exists());
4367        let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4368        let proof: CargoResidue =
4369            serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4370        assert_eq!(
4371            proof.files[Path::new("debug/output.d")],
4372            std::fs::read(wt.join("target/debug/output.d")).unwrap()
4373        );
4374        std::fs::write(
4375            changed.candidates[0].worktree.join("target/debug/output.d"),
4376            "owner notes",
4377        )
4378        .unwrap();
4379        std::fs::write(
4380            unknown.candidates[0].worktree.join("target/debug/notes"),
4381            "keep",
4382        )
4383        .unwrap();
4384        Fx::target(&unproven.candidates[0].worktree);
4385        std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4386        disk.auto_fold = true;
4387        let locks = cargo_locks(&wt.join("target")).unwrap();
4388        let mut report = Report::default();
4389        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4390        assert_eq!(report.retired, 0, "{report:?}");
4391        assert!(
4392            report
4393                .retained
4394                .iter()
4395                .any(|reason| reason.contains("Cargo build"))
4396        );
4397        drop(locks);
4398        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4399        assert_eq!(report.retired, 1, "{report:?}");
4400        assert!(!wt.exists());
4401        assert!(!reviewer.exists());
4402        assert!(
4403            residue_proof(&f.home, &empty.id, &reviewer)
4404                .unwrap()
4405                .exists()
4406        );
4407        assert!(
4408            proof_path.exists(),
4409            "metadata evidence outlives the worktree"
4410        );
4411        assert_eq!(
4412            RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4413            Some(source.id)
4414        );
4415        for run in [&changed, &unknown, &unproven] {
4416            assert!(run.candidates[0].worktree.exists());
4417            assert!(
4418                RunState::load_under(&run.id, &f.home)
4419                    .unwrap()
4420                    .retired_to
4421                    .is_none()
4422            );
4423        }
4424    }
4425
4426    #[tokio::test]
4427    async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4428        let f = Fx::new().await;
4429        let source = f.run("0000", true).await;
4430        let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4431            .await
4432            .unwrap();
4433        let mut runs = vec![source.clone()];
4434        for n in 1..=12 {
4435            runs.push(f.run(&format!("{n:04}"), false).await);
4436        }
4437        let dirty = runs[1].candidates[0].worktree.clone();
4438        std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4439        let ignored = runs[2].candidates[0].worktree.clone();
4440        std::fs::create_dir(ignored.join("scratch")).unwrap();
4441        std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4442        // A missing candidate array does not prove that an unrecorded worktree
4443        // is empty: discovery must inspect Git and the worktree itself.
4444        runs[1].candidates.clear();
4445        runs[1].save_under(&f.home).unwrap();
4446        let task = f.task(&runs);
4447        let mut total = 0;
4448        for _ in 0..4 {
4449            let mut report = Report::default();
4450            retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4451            assert!(report.retired <= 5);
4452            total += report.retired;
4453        }
4454        assert_eq!(total, 10);
4455        assert!(dirty.exists());
4456        assert!(ignored.exists());
4457        assert_eq!(
4458            git::rev_parse(&f.repo, &source.candidates[0].branch)
4459                .await
4460                .unwrap(),
4461            source_head
4462        );
4463        let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4464        assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4465        assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4466        let mut released = task.clone();
4467        released.release();
4468        assert_eq!(
4469            crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4470                RunState::load_under(id, &f.home)
4471            }),
4472            Some(source.id.clone())
4473        );
4474
4475        assert!(
4476            f.home
4477                .join("runs")
4478                .join(&retired.id)
4479                .join("run.json")
4480                .exists()
4481        );
4482        assert!(
4483            f.home
4484                .join("recovery-lineages")
4485                .join(format!("{}.json", task.id))
4486                .exists()
4487        );
4488    }
4489
4490    #[tokio::test]
4491    async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4492        let f = Fx::new().await;
4493        let a = f.run("dddd", true).await;
4494        let b = f.run("eeee", true).await;
4495        let empty = f.run("ffff", false).await;
4496        let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4497        assert!(
4498            crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4499            "unfinished queue history protects older source branches from ordinary folding"
4500        );
4501        let mut prior = a.clone();
4502        assert!(
4503            crate::graph::fold_run(&mut prior, true, &f.home)
4504                .await
4505                .unwrap_err()
4506                .to_string()
4507                .contains(&task.id)
4508        );
4509        let mut report = Report::default();
4510        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4511        assert_eq!(report.retired, 0);
4512        assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4513        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4514        Fx::target(&a.candidates[0].worktree);
4515        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4516        assert!(
4517            report.freed_bytes > 0,
4518            "source retention does not retain regenerable outputs"
4519        );
4520        assert!(a.candidates[0].worktree.exists());
4521        assert!(b.candidates[0].worktree.exists());
4522        assert!(empty.candidates[0].worktree.exists());
4523    }
4524
4525    #[tokio::test]
4526    async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4527        let f = Fx::new().await;
4528        let authority = f.run("p000", true).await;
4529        let question_run = f.run("p001", false).await;
4530        let mut checkpoint = f.run("p002", false).await;
4531        checkpoint.land_approval = Some(crate::run::LandApproval {
4532            question: "pending".into(),
4533            head: f.base.clone(),
4534        });
4535        checkpoint.save_under(&f.home).unwrap();
4536        let retained = f.run("p003", false).await;
4537        std::fs::write(
4538            f.home
4539                .join("runs")
4540                .join(&retained.id)
4541                .join("retain-recovery-sources"),
4542            "owner",
4543        )
4544        .unwrap();
4545        let mut partial = f.run("p004", false).await;
4546        let reviewer = partial.worktree_root().join("review-1");
4547        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4548            .await
4549            .unwrap();
4550        // Model a crash after the retirement fence but only one removal.
4551        partial.retired_to = Some(authority.id.clone());
4552        partial.save_under(&f.home).unwrap();
4553        assert!(
4554            git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4555                .await
4556                .unwrap()
4557        );
4558        let git_checkpoint = f.run("p005", false).await;
4559        let location = git::git(
4560            &git_checkpoint.candidates[0].worktree,
4561            &[
4562                "rev-parse",
4563                "--path-format=absolute",
4564                "--git-path",
4565                "sequencer",
4566            ],
4567        )
4568        .await
4569        .unwrap();
4570        std::fs::create_dir(location.trim()).unwrap();
4571        let mut preservation = f.run("p006", false).await;
4572        preservation
4573            .candidate_preservation
4574            .insert('A', "pending preservation".into());
4575        preservation.save_under(&f.home).unwrap();
4576        let task = f.task(&[
4577            preservation.clone(),
4578            git_checkpoint.clone(),
4579            authority.clone(),
4580            question_run.clone(),
4581            checkpoint.clone(),
4582            retained.clone(),
4583            partial.clone(),
4584        ]);
4585        let questions = crate::ask::Questions::at(f.home.join("questions"));
4586        let mut question = crate::ask::Question::new(
4587            question_run.id.clone(),
4588            "implement".into(),
4589            "fixture".into(),
4590            "owner decision".into(),
4591            String::new(),
4592            vec!["keep".into()],
4593        );
4594        questions.put(&mut question).unwrap();
4595        let mut report = Report::default();
4596        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4597        assert_eq!(report.retired, 1, "{report:?}");
4598        assert!(!reviewer.exists());
4599        for state in [
4600            &question_run,
4601            &checkpoint,
4602            &retained,
4603            &git_checkpoint,
4604            &preservation,
4605        ] {
4606            assert!(state.candidates[0].worktree.exists());
4607            assert!(
4608                RunState::load_under(&state.id, &f.home)
4609                    .unwrap()
4610                    .retired_to
4611                    .is_none()
4612            );
4613        }
4614        assert!(questions.get(&question.id).unwrap().status.open());
4615        assert_eq!(
4616            crate::queue::Queue::at(f.home.join("queue"))
4617                .get(&task.id)
4618                .unwrap()
4619                .status,
4620            task.status
4621        );
4622    }
4623
4624    #[tokio::test]
4625    async fn preservation_schema_migration_remains_distinct_from_retirement() {
4626        let f = Fx::new().await;
4627        let mut state = f.run("p007", false).await;
4628        state.schema = 15;
4629        state.save_under(&f.home).unwrap();
4630        let native = RunState::load_under(&state.id, &f.home).unwrap();
4631        assert_eq!(native.schema, crate::run::SCHEMA);
4632        assert!(native.candidate_preservation.is_empty());
4633        assert!(native.retired_to.is_none());
4634        state.schema = 14;
4635        state.save_under(&f.home).unwrap();
4636        let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4637        assert!(legacy.candidate_preservation.contains_key(&'A'));
4638        legacy.save_under(&f.home).unwrap();
4639        let restored = RunState::load_under(&state.id, &f.home).unwrap();
4640        assert_eq!(
4641            restored.candidate_preservation,
4642            legacy.candidate_preservation
4643        );
4644    }
4645
4646    #[tokio::test]
4647    async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4648        let f = Fx::new().await;
4649        let cache = f.repo.join("cache");
4650        let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4651        std::fs::create_dir_all(&cache).unwrap();
4652        assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4653        assert!(
4654            crate::cache::maintenance_prune(&f.home, &cache, 1)
4655                .unwrap()
4656                .is_none()
4657        );
4658        drop(guard);
4659        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4660    }
4661
4662    #[tokio::test]
4663    async fn launcher_death_does_not_release_live_tool_output_reservations() {
4664        let f = Fx::new().await;
4665        let run = f.run("abcd", false).await;
4666        let wt = &run.candidates[0].worktree;
4667        let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4668        helper
4669            .quiet()
4670            .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4671            .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4672            .env("MAGI_TEST_WORKTREE", wt)
4673            .stdin(std::process::Stdio::null())
4674            .stdout(std::process::Stdio::null())
4675            .stderr(std::process::Stdio::null())
4676            .kill_on_drop(true);
4677        assert!(
4678            tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4679                .await
4680                .unwrap()
4681                .unwrap()
4682                .success()
4683        );
4684        let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4685            .unwrap()
4686            .trim()
4687            .parse()
4688            .unwrap();
4689        struct KillTool(i32);
4690        impl Drop for KillTool {
4691            fn drop(&mut self) {
4692                // SAFETY: PID belongs to the bounded helper above.
4693                unsafe { libc::kill(self.0, libc::SIGKILL) };
4694            }
4695        }
4696        let tool = KillTool(pid);
4697        assert!(OutputLease::take(wt, true).is_err());
4698        assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4699        drop(tool);
4700        tokio::time::timeout(std::time::Duration::from_secs(5), async {
4701            loop {
4702                if OutputLease::take(wt, true).is_ok()
4703                    && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4704                {
4705                    break;
4706                }
4707                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4708            }
4709        })
4710        .await
4711        .unwrap();
4712    }
4713
4714    #[tokio::test]
4715    async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4716        let f = Fx::new().await;
4717        let run = f.run("abcd", false).await;
4718        let wt = &run.candidates[0].worktree;
4719        let cache = f.repo.join("cache");
4720        std::fs::create_dir(&cache).unwrap();
4721        for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4722            let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4723            let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4724            cmd.quiet()
4725                .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4726                .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4727                .stdin(std::process::Stdio::null())
4728                .stdout(std::process::Stdio::null())
4729                .stderr(std::process::Stdio::null())
4730                .kill_on_drop(true);
4731            if clear_owner {
4732                cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4733            } else {
4734                cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4735            }
4736            let gib = 1024 * 1024 * 1024;
4737            let mut leases = vec![
4738                OutputLease::borrow(wt).unwrap().unwrap(),
4739                OutputLease::cache(&f.home, &cache, false).unwrap(),
4740            ];
4741            leases.extend(
4742                OutputLease::admit_with(
4743                    &f.home,
4744                    std::slice::from_ref(wt),
4745                    8 * gib,
4746                    |_| Ok(9 * gib),
4747                )
4748                .unwrap(),
4749            );
4750            let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4751            assert!(
4752                tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4753                    .await
4754                    .unwrap()
4755                    .unwrap()
4756                    .success()
4757            );
4758            let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4759            struct KillTool(Option<i32>);
4760            impl Drop for KillTool {
4761                fn drop(&mut self) {
4762                    // SAFETY: PID from our helper; kill also bounds panic paths.
4763                    if let Some(pid) = self.0 {
4764                        unsafe { libc::kill(pid, libc::SIGKILL) };
4765                    }
4766                }
4767            }
4768            let mut tool = KillTool(Some(pid));
4769            // The CLI has already exited and cannot supply a parent-tree
4770            // relationship. Either the witness or inherited environment must
4771            // retain ownership; test each independently.
4772            assert!(OutputLease::take(wt, true).is_err());
4773            let finish = if cancel {
4774                drop(running);
4775                None
4776            } else {
4777                Some(tokio::spawn(async move { running.finish().await }))
4778            };
4779            if clear_owner {
4780                tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4781                assert!(crate::proc::pid_alive(pid as u32));
4782                assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4783                assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4784                assert!(
4785                    OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4786                        9 * gib
4787                    ))
4788                    .is_err(),
4789                    "orphan still owns its disk reservation"
4790                );
4791                assert!(
4792                    crate::cache::maintenance_prune(&f.home, &cache, 0)
4793                        .unwrap()
4794                        .is_none()
4795                );
4796                if let Some(finish) = &finish {
4797                    assert!(!finish.is_finished());
4798                }
4799                // Witness-only ownership waits for natural/external tool exit.
4800                unsafe { libc::kill(pid, libc::SIGKILL) };
4801            }
4802            if let Some(finish) = finish {
4803                tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4804                    .await
4805                    .unwrap()
4806                    .unwrap();
4807            }
4808            tokio::time::timeout(std::time::Duration::from_secs(5), async {
4809                loop {
4810                    if OutputLease::take(wt, true).is_ok()
4811                        && OutputLease::cache(&f.home, &cache, true).is_ok()
4812                        && OutputLease::admit_with(
4813                            &f.home,
4814                            std::slice::from_ref(wt),
4815                            8 * gib,
4816                            |_| Ok(9 * gib),
4817                        )
4818                        .is_ok()
4819                    {
4820                        // Token-owned orphans must be stopped by cleanup itself.
4821                        if !clear_owner {
4822                            assert!(!crate::proc::pid_alive(pid as u32));
4823                        }
4824                        break;
4825                    }
4826                    tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4827                }
4828            })
4829            .await
4830            .unwrap();
4831            tool.0 = None;
4832        }
4833    }
4834
4835    #[tokio::test]
4836    async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4837        let f = Fx::new().await;
4838        let run = f.run("abcd", false).await;
4839        let wt = &run.candidates[0].worktree;
4840        let cache = f.repo.join("cache");
4841        std::fs::create_dir(&cache).unwrap();
4842        let pid_file = f.repo.join("child-pid");
4843        let mut cmd = tokio::process::Command::new("sh");
4844        cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4845            .quiet()
4846            .env("CHILD_PID_FILE", &pid_file)
4847            .process_group(0)
4848            .kill_on_drop(true);
4849        let running = crate::seat_process::Running::spawn(
4850            cmd,
4851            None,
4852            vec![
4853                OutputLease::borrow(wt).unwrap().unwrap(),
4854                OutputLease::cache(&f.home, &cache, false).unwrap(),
4855            ],
4856        )
4857        .unwrap();
4858        tokio::time::timeout(std::time::Duration::from_secs(5), async {
4859            while !pid_file.exists() {
4860                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4861            }
4862        })
4863        .await
4864        .unwrap();
4865        assert!(OutputLease::take(wt, true).is_err());
4866        assert!(
4867            crate::cache::maintenance_prune(&f.home, &cache, 0)
4868                .unwrap()
4869                .is_none()
4870        );
4871        let pid: u32 = std::fs::read_to_string(pid_file)
4872            .unwrap()
4873            .trim()
4874            .parse()
4875            .unwrap();
4876        drop(running);
4877        tokio::time::timeout(std::time::Duration::from_secs(10), async {
4878            loop {
4879                if let Ok(guard) = OutputLease::take(wt, true) {
4880                    assert!(
4881                        !crate::proc::pid_alive(pid),
4882                        "output lock released before tool death"
4883                    );
4884                    drop(guard);
4885                    break;
4886                }
4887                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4888            }
4889        })
4890        .await
4891        .unwrap();
4892        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4893    }
4894
4895    #[tokio::test]
4896    async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
4897        let f = Fx::new().await;
4898        let reason = tokio::time::timeout(
4899            std::time::Duration::from_secs(2),
4900            emergency(vec![f.repo.clone()], u64::MAX),
4901        )
4902        .await
4903        .unwrap();
4904        assert!(reason.contains("disk pressure"));
4905        assert!(
4906            tokio::time::timeout(
4907                std::time::Duration::from_millis(10),
4908                emergency(vec![f.repo.clone()], 0)
4909            )
4910            .await
4911            .is_err()
4912        );
4913        assert_eq!(
4914            std::fs::read_to_string(f.repo.join("source")).unwrap(),
4915            "base\n"
4916        );
4917    }
4918
4919    #[tokio::test]
4920    async fn automatic_requeues_cannot_refund_failed_input_generation() {
4921        let f = Fx::new().await;
4922        let mut run = f.run("1234", false).await;
4923        run.prep_failure = Some("cherry-pick conflict".into());
4924        let mut task = f.task(std::slice::from_ref(&run));
4925        let inputs = crate::queue::recovery_inputs(&run.sources);
4926        for _ in 0..108 {
4927            task.recovery_failed(&run);
4928            task.requeue();
4929            assert_eq!(task.attempts, 0);
4930        }
4931        let bytes = serde_json::to_vec(&task).unwrap();
4932        let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
4933        assert!(
4934            restored
4935                .recovery_exhausted(&run.base_commit, &inputs, 2)
4936                .is_some()
4937        );
4938        assert!(
4939            restored
4940                .recovery_exhausted("different-base", &inputs, 2)
4941                .is_none()
4942        );
4943        let original_base = run.base_commit.clone();
4944        run.base_commit = "different-base".into();
4945        restored.recovery_failed(&run);
4946        assert!(
4947            restored
4948                .recovery_exhausted(&original_base, &inputs, 2)
4949                .is_some(),
4950            "returning to old inputs does not replenish their budget"
4951        );
4952        assert!(
4953            restored
4954                .recovery_exhausted(&run.base_commit, &inputs, 2)
4955                .is_none()
4956        );
4957        restored.release_operator("test-owner", None);
4958        assert!(
4959            restored
4960                .recovery_exhausted(&run.base_commit, &inputs, 2)
4961                .is_none()
4962        );
4963    }
4964    async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
4965        let mut state = f.run(tag, useful).await;
4966        state.status = RunStatus::Superseded;
4967        state.save_under(&f.home).unwrap();
4968        state
4969    }
4970
4971    fn historical_review(state: &mut RunState, head: &str) {
4972        state.reviews.push(
4973            serde_json::from_value(serde_json::json!({
4974                "round": state.reviews.len() + 1, "head": head, "reviews": []
4975            }))
4976            .unwrap(),
4977        );
4978    }
4979
4980    async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
4981        let path = state.worktree_root().join("review-1");
4982        git::worktree_add_detached(&f.repo, &path, head)
4983            .await
4984            .unwrap();
4985        path
4986    }
4987
4988    #[tokio::test]
4989    async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
4990        use std::os::unix::fs::PermissionsExt;
4991        let mut f = Fx::new().await;
4992        std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
4993        git::git(&f.repo, &["add", "."]).await.unwrap();
4994        git::git(&f.repo, &["commit", "-m", "binary base"])
4995            .await
4996            .unwrap();
4997        f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
4998        let mut state = inactive_source(&f, "ec01", true).await;
4999        let candidate = state.candidates[0].worktree.clone();
5000        let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5001        historical_review(&mut state, &earlier);
5002        let review = reviewer(&f, &state, &earlier).await;
5003        std::fs::remove_file(candidate.join("source")).unwrap();
5004        std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5005        std::fs::write(candidate.join("script"), "run\n").unwrap();
5006        std::fs::set_permissions(
5007            candidate.join("script"),
5008            std::fs::Permissions::from_mode(0o755),
5009        )
5010        .unwrap();
5011        git::git(&candidate, &["add", "."]).await.unwrap();
5012        git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5013            .await
5014            .unwrap();
5015        let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5016        let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5017        let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5018            .await
5019            .unwrap();
5020        historical_review(&mut state, &head);
5021        historical_review(&mut state, &head);
5022        state.status = RunStatus::Superseded;
5023        state.save_under(&f.home).unwrap();
5024        let task = f.task(std::slice::from_ref(&state));
5025        let before = serde_json::to_value(&state).unwrap();
5026        git::git(&review, &["reset", "--hard", &head])
5027            .await
5028            .unwrap();
5029        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5030        assert_eq!(report.evicted_copies, 1, "{report:?}");
5031        assert!(!review.exists());
5032        assert!(candidate.exists(), "selected implementation survives");
5033        assert!(report.source_unlinked_logical_bytes > 0);
5034        assert!(report.source_unlinked_allocated_bytes.is_some());
5035        assert!(report.source_evidence_added_allocated_bytes.is_some());
5036        assert_eq!(
5037            serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5038            before
5039        );
5040        assert_eq!(
5041            crate::queue::Queue::at(f.home.join("queue"))
5042                .get(&task.id)
5043                .unwrap()
5044                .runs,
5045            task.runs
5046        );
5047        let archive = f
5048            .home
5049            .join("runs")
5050            .join(&state.id)
5051            .join("artifacts/source-evidence")
5052            .join(format!("{head}-{}", f.base));
5053        let patch = source_diff(&archive).await.unwrap();
5054        assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5055        assert!(String::from_utf8_lossy(&patch).contains("100755"));
5056        assert!(
5057            git::worktree_remove_clean(&f.repo, &candidate)
5058                .await
5059                .unwrap()
5060        );
5061        git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5062            .await
5063            .unwrap();
5064        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5065            .await
5066            .unwrap();
5067        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5068        assert!(
5069            !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5070                .await
5071                .unwrap()
5072                .ok()
5073        );
5074        // Even the remaining base repository is unavailable to the reader.
5075        std::fs::remove_dir_all(&f.repo).unwrap();
5076        assert_eq!(source_diff(&archive).await.unwrap(), patch);
5077        let restored = f._temp.path().join("restored");
5078        restore_source(&archive, &restored).await.unwrap();
5079        assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5080        assert_eq!(
5081            std::fs::read(restored.join("binary")).unwrap(),
5082            b"new\0binary\xfe"
5083        );
5084        assert!(!restored.join("source").exists());
5085        assert_ne!(
5086            std::fs::metadata(restored.join("script"))
5087                .unwrap()
5088                .permissions()
5089                .mode()
5090                & 0o111,
5091            0
5092        );
5093        assert!(restore_source(&archive, &restored).await.is_err());
5094        // Reconstruct from the archived base plus the displayed binary patch,
5095        // rather than relying only on checking out the archived final tree.
5096        git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5097            .await
5098            .unwrap();
5099        let patch_file = f._temp.path().join("saved.patch");
5100        std::fs::write(&patch_file, &patch).unwrap();
5101        git::git(
5102            &restored,
5103            &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5104        )
5105        .await
5106        .unwrap();
5107        assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5108        assert_eq!(
5109            std::fs::read(restored.join("binary")).unwrap(),
5110            b"new\0binary\xfe"
5111        );
5112    }
5113
5114    #[tokio::test]
5115    async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5116        let f = Fx::new().await;
5117        let state = inactive_source(&f, "ec10", true).await;
5118        let review = reviewer(&f, &state, &f.base).await;
5119        let mut successor = inactive_source(&f, "ec11", true).await;
5120        successor.retired_to = Some(state.id.clone());
5121        successor.save_under(&f.home).unwrap();
5122        f.task(&[state.clone(), successor]);
5123
5124        let mut tips = Vec::new();
5125        for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5126            std::fs::write(review.join("source"), contents).unwrap();
5127            git::git(&review, &["add", "."]).await.unwrap();
5128            let tree = git::git(&review, &["write-tree"]).await.unwrap();
5129            let tip = git::git(
5130                &review,
5131                &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5132            )
5133            .await
5134            .unwrap();
5135            git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5136            tips.push(tip);
5137            git::git(&review, &["reset", "--hard", &f.base])
5138                .await
5139                .unwrap();
5140        }
5141        // Keep one tip exclusively in ORIG_HEAD, the other in the HEAD log.
5142        git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5143            .await
5144            .unwrap();
5145        git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5146            .await
5147            .unwrap();
5148        let mut report = Report::default();
5149        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5150        assert_eq!(report.evicted_copies, 1, "{report:?}");
5151        assert!(!review.exists());
5152        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5153            .await
5154            .unwrap();
5155        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5156        for tip in &tips {
5157            assert!(
5158                !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5159                    .await
5160                    .unwrap()
5161                    .ok()
5162            );
5163        }
5164        // The run-oriented interface also works without the source object store.
5165        std::fs::remove_dir_all(&f.repo).unwrap();
5166        let shown = seat_evidence(&f.home, &state.id).await.unwrap();
5167        for (i, tip) in tips.iter().enumerate() {
5168            assert!(
5169                shown
5170                    .iter()
5171                    .any(|row| row.contains(tip) && row.contains("patch_bytes="))
5172            );
5173            let destination = f._temp.path().join(format!("restore-history-{i}"));
5174            restore_seat(&f.home, &state.id, tip, &destination)
5175                .await
5176                .unwrap();
5177            assert_eq!(
5178                std::fs::read_to_string(destination.join("source")).unwrap(),
5179                ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
5180            );
5181        }
5182        assert!(
5183            restore_seat(
5184                &f.home,
5185                &state.id,
5186                "unknown",
5187                &f._temp.path().join("missing")
5188            )
5189            .await
5190            .is_err()
5191        );
5192    }
5193
5194    #[tokio::test]
5195    async fn source_eviction_revalidates_archive_immediately_before_unlink() {
5196        fn corrupt(path: &Path) -> Result<()> {
5197            let root = path.parent().unwrap();
5198            // Locate the fixture's durable artifact store through its run record
5199            // path supplied below by a test-only hook in the artifact tree.
5200            let marker = root.join("archive-to-corrupt");
5201            let archive = std::fs::read_to_string(&marker)?;
5202            std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
5203            Ok(())
5204        }
5205        let f = Fx::new().await;
5206        let state = inactive_source(&f, "ec12", true).await;
5207        let review = reviewer(&f, &state, &f.base).await;
5208        let archive = f
5209            .home
5210            .join("runs")
5211            .join(&state.id)
5212            .join("artifacts/source-evidence")
5213            .join(format!("{}-{}", f.base, f.base));
5214        std::fs::write(
5215            state.worktree_root().join("archive-to-corrupt"),
5216            archive.to_str().unwrap(),
5217        )
5218        .unwrap();
5219        let mut report = Report::default();
5220        copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
5221        assert_eq!(report.evicted_copies, 0, "{report:?}");
5222        assert!(review.exists());
5223        assert!(
5224            report
5225                .retained
5226                .iter()
5227                .any(|r| r.contains("corrupt snapshot")),
5228            "{report:?}"
5229        );
5230    }
5231
5232    #[tokio::test]
5233    async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
5234        for kind in [
5235            "dirty",
5236            "hidden",
5237            "ignored",
5238            "empty-directory",
5239            "untracked",
5240            "borrowed",
5241            "lease",
5242            "owner",
5243            "driver",
5244            "task-pin",
5245            "task-replacement",
5246            "malformed-orig-head",
5247            "grace",
5248            "resumable",
5249            "archive-symlink",
5250            "root-symlink",
5251            "public",
5252            "checkpoint",
5253            "symlink",
5254            "retained",
5255            "unreadable",
5256            "unknown-field",
5257            "unknown-task-field",
5258        ] {
5259            let f = Fx::new().await;
5260            let mut state = inactive_source(&f, "ec02", true).await;
5261            let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5262                .await
5263                .unwrap();
5264            historical_review(&mut state, &head);
5265            let review = reviewer(&f, &state, &head).await;
5266            let mut output = None;
5267            let mut owner = None;
5268            let mut driver = None;
5269            let mut disk = Fx::disk();
5270            match kind {
5271                "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
5272                "hidden" => {
5273                    git::git(&review, &["update-index", "--assume-unchanged", "source"])
5274                        .await
5275                        .unwrap();
5276                    std::fs::write(review.join("source"), "notes").unwrap();
5277                }
5278                "ignored" => {
5279                    std::fs::create_dir(review.join("scratch")).unwrap();
5280                    std::fs::write(review.join("scratch/notes"), "notes").unwrap();
5281                }
5282                "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
5283                "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
5284                "borrowed" => {
5285                    let mut other = f.run("ec03", true).await;
5286                    other.candidates[0].worktree = review.clone();
5287                    other.save_under(&f.home).unwrap();
5288                }
5289                "lease" => output = OutputLease::borrow(&review).unwrap(),
5290                "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
5291                "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
5292                "task-pin" => {
5293                    let mut task = f.task(std::slice::from_ref(&state));
5294                    task.resume_override = Some(crate::queue::OperatorResume {
5295                        question_id: "fixture-pin".into(),
5296                        at: Fx::now(),
5297                        conductor_rehold: None,
5298                        forced: true,
5299                        pinned_run: Some(state.id.clone()),
5300                    });
5301                    crate::queue::Queue::at(f.home.join("queue"))
5302                        .put(&mut task)
5303                        .unwrap();
5304                }
5305                "task-replacement" => {
5306                    let mut task = f.task(std::slice::from_ref(&state));
5307                    task.replacement = Some(crate::merge_decision::Replacement {
5308                        question: "fixture-replacement".into(),
5309                        run: state.id.clone(),
5310                        pr: "https://example.invalid/pull/1".into(),
5311                        agent: "worker".into(),
5312                        context: String::new(),
5313                    });
5314                    crate::queue::Queue::at(f.home.join("queue"))
5315                        .put(&mut task)
5316                        .unwrap();
5317                }
5318                "malformed-orig-head" => {
5319                    let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
5320                        .await
5321                        .unwrap();
5322                    std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
5323                        .unwrap();
5324                }
5325                "grace" => disk.fold_grace_secs = 3600,
5326                "resumable" => state.status = RunStatus::Blocked,
5327                "archive-symlink" => {
5328                    let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
5329                    std::fs::create_dir_all(&artifacts).unwrap();
5330                    let outside = f._temp.path().join("outside");
5331                    std::fs::create_dir(&outside).unwrap();
5332                    std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
5333                        .unwrap();
5334                }
5335                "root-symlink" => {
5336                    let root = state.worktree_root();
5337                    let outside = f._temp.path().join("outside-root");
5338                    std::fs::rename(&root, &outside).unwrap();
5339                    std::os::unix::fs::symlink(&outside, &root).unwrap();
5340                }
5341                "public" => state.pr_head = Some(state.candidates[0].branch.clone()),
5342                "checkpoint" => {
5343                    let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
5344                        .await
5345                        .unwrap();
5346                    std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
5347                }
5348                "symlink" => {
5349                    std::fs::remove_file(review.join("source")).unwrap();
5350                    std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
5351                }
5352                "retained" => std::fs::write(
5353                    f.home
5354                        .join("runs")
5355                        .join(&state.id)
5356                        .join("retain-recovery-sources"),
5357                    "owner",
5358                )
5359                .unwrap(),
5360                "unreadable" => {
5361                    std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
5362                }
5363                "unknown-field" | "unknown-task-field" => {}
5364                _ => unreachable!(),
5365            }
5366            state.save_under(&f.home).unwrap();
5367            if kind == "unknown-field" {
5368                let path = f.home.join("runs").join(&state.id).join("run.json");
5369                let mut raw: serde_json::Value =
5370                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5371                raw["future_source_owner"] = serde_json::json!({"path": review});
5372                std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
5373            }
5374            if kind == "unknown-task-field" {
5375                let task = f.task(std::slice::from_ref(&state));
5376                let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
5377                let mut raw: serde_json::Value =
5378                    serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5379                raw["future_source_owner"] = serde_json::json!({"path": review});
5380                std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
5381            }
5382            let mut report = Report::default();
5383            copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
5384            assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5385            assert!(review.exists(), "{kind}: {report:?}");
5386            assert!(!report.retained.is_empty(), "{kind}: {report:?}");
5387            drop(output);
5388            drop(owner);
5389            drop(driver);
5390        }
5391    }
5392
5393    #[tokio::test]
5394    async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
5395        let f = Fx::new().await;
5396        let state = inactive_source(&f, "ec04", true).await;
5397        let review = reviewer(&f, &state, &f.base).await;
5398        let mut report = Report::default();
5399        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5400        assert_eq!(report.evicted_copies, 1, "{report:?}");
5401        let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
5402        let ledger = artifacts.join("eviction-review-1.json");
5403        let mut record: serde_json::Value =
5404            serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
5405        record["phase"] = "pending".into();
5406        std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
5407        let archive = PathBuf::from(record["evidence"].as_str().unwrap());
5408        let pack = std::fs::read(archive.join("objects.pack")).unwrap();
5409        std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
5410        let mut blocked = Report::default();
5411        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
5412        assert!(
5413            blocked
5414                .retained
5415                .iter()
5416                .any(|r| r.contains("corrupt snapshot")),
5417            "{blocked:?}"
5418        );
5419        assert_eq!(
5420            serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
5421                ["phase"],
5422            "pending"
5423        );
5424        assert!(
5425            restore_source(&archive, &f._temp.path().join("bad-restore"))
5426                .await
5427                .is_err()
5428        );
5429        std::fs::write(archive.join("objects.pack"), &pack).unwrap();
5430        let mut retried = Report::default();
5431        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
5432        assert_eq!(
5433            serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
5434                ["phase"],
5435            "evicted"
5436        );
5437        assert_eq!(retried.evicted_copies, 0);
5438        assert!(!review.exists());
5439        // A stopped archive publisher can resume its immutable staging files.
5440        let pending = archive.with_extension("pending");
5441        std::fs::rename(&archive, &pending).unwrap();
5442        assert_eq!(
5443            crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
5444                .await
5445                .unwrap(),
5446            archive
5447        );
5448        assert!(!pending.exists());
5449    }
5450
5451    #[tokio::test]
5452    async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
5453        let f = Fx::new().await;
5454        let mut empty = inactive_source(&f, "ec05", false).await;
5455        historical_review(&mut empty, &f.base);
5456        empty.status = RunStatus::Superseded;
5457        empty.save_under(&f.home).unwrap();
5458        let source = inactive_source(&f, "ec06", true).await;
5459        let task = f.task(&[empty.clone(), source.clone()]);
5460        let before = serde_json::to_value(&empty).unwrap();
5461        let mut report = Report::default();
5462        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5463        assert_eq!(report.evicted_copies, 1, "{report:?}");
5464        assert!(!empty.candidates[0].worktree.exists());
5465        assert!(source.candidates[0].worktree.exists());
5466        assert!(
5467            git::branch_exists(&f.repo, &empty.candidates[0].branch)
5468                .await
5469                .unwrap()
5470        );
5471        assert_eq!(
5472            serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
5473            before
5474        );
5475        let mut retry = Report::default();
5476        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5477        assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
5478        assert!(!empty.worktree_root().exists());
5479        assert_eq!(
5480            crate::queue::Queue::at(f.home.join("queue"))
5481                .get(&task.id)
5482                .unwrap()
5483                .runs,
5484            task.runs
5485        );
5486    }
5487    #[tokio::test]
5488    async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
5489        fn interrupt(_: &Path) -> Result<()> {
5490            anyhow::bail!("fixture interruption before unlink")
5491        }
5492        fn vanish(path: &Path) -> Result<()> {
5493            std::fs::remove_dir_all(path)?;
5494            anyhow::bail!("fixture interruption after checkout unlink")
5495        }
5496        for stage in [
5497            "intent",
5498            "missing-registration",
5499            "changed-admin",
5500            "missing-archive",
5501            "wrong-identity",
5502        ] {
5503            let f = Fx::new().await;
5504            let state = inactive_source(&f, "ec07", true).await;
5505            let review = reviewer(&f, &state, &f.base).await;
5506            let admin = PathBuf::from(
5507                git::git(&review, &["rev-parse", "--absolute-git-dir"])
5508                    .await
5509                    .unwrap(),
5510            );
5511            let mut interrupted = Report::default();
5512            copies::evict_with(
5513                &f.home,
5514                &Fx::disk(),
5515                Fx::now(),
5516                &mut interrupted,
5517                if stage == "missing-registration" || stage == "changed-admin" {
5518                    vanish
5519                } else {
5520                    interrupt
5521                },
5522            )
5523            .await;
5524            assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
5525            let ledger = f
5526                .home
5527                .join("runs")
5528                .join(&state.id)
5529                .join("artifacts/eviction-review-1.json");
5530            let record: serde_json::Value =
5531                serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
5532            assert_eq!(record["phase"], "pending");
5533            if stage == "changed-admin" {
5534                std::fs::write(admin.join("owner-note"), "keep").unwrap();
5535            }
5536            if stage == "missing-archive" {
5537                std::fs::remove_file(
5538                    Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
5539                )
5540                .unwrap();
5541            }
5542            if stage == "wrong-identity" {
5543                let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5544                    .await
5545                    .unwrap();
5546                let other =
5547                    crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
5548                        .await
5549                        .unwrap();
5550                let original = Path::new(record["evidence"].as_str().unwrap());
5551                for name in [
5552                    "manifest.json",
5553                    "objects.pack",
5554                    "change.patch",
5555                    "change.index",
5556                ] {
5557                    std::fs::copy(other.join(name), original.join(name)).unwrap();
5558                }
5559            }
5560            let mut retried = Report::default();
5561            copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
5562            let phase =
5563                serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
5564                    .unwrap()["phase"]
5565                    .clone();
5566            if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
5567                assert_eq!(phase, "pending", "{stage}: {retried:?}");
5568                assert!(!retried.retained.is_empty());
5569                if stage == "changed-admin" {
5570                    assert!(admin.join("owner-note").exists());
5571                } else {
5572                    assert!(review.exists());
5573                }
5574            } else {
5575                assert_eq!(phase, "evicted", "{stage}: {retried:?}");
5576                assert!(!review.exists());
5577                assert!(!admin.exists());
5578            }
5579        }
5580    }
5581    #[tokio::test]
5582    async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
5583        let f = Fx::new().await;
5584        let mut runs = Vec::new();
5585        let mut paths = Vec::new();
5586        for tag in ["ec10", "ec11", "ec12", "ec13"] {
5587            let state = inactive_source(&f, tag, true).await;
5588            for name in ["judge-1", "advisor-1"] {
5589                let path = state.worktree_root().join(name);
5590                git::worktree_add_detached(&f.repo, &path, &f.base)
5591                    .await
5592                    .unwrap();
5593                paths.push(path);
5594            }
5595            runs.push(state);
5596        }
5597        let task = f.task(&runs);
5598        let mut report = Report::default();
5599        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5600        assert_eq!(report.evicted_copies, 5, "{report:?}");
5601        let mut next = Report::default();
5602        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
5603        assert_eq!(next.evicted_copies, 3, "{next:?}");
5604        assert!(paths.iter().all(|p| !p.exists()));
5605        assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
5606        assert_eq!(
5607            crate::queue::Queue::at(f.home.join("queue"))
5608                .get(&task.id)
5609                .unwrap()
5610                .runs,
5611            task.runs
5612        );
5613        let limited = Fx::new().await;
5614        let state = inactive_source(&limited, "ec15", true).await;
5615        let copy = reviewer(&limited, &state, &limited.base).await;
5616        let evidence = limited
5617            .home
5618            .join("runs")
5619            .join(&state.id)
5620            .join("artifacts/source-evidence");
5621        std::fs::create_dir_all(&evidence).unwrap();
5622        let retained = evidence.join("reserved-evidence");
5623        std::fs::File::create(&retained)
5624            .unwrap()
5625            .set_len(64 * 1024 * 1024)
5626            .unwrap();
5627        let mut blocked = Report::default();
5628        copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
5629        assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
5630        assert!(copy.exists());
5631        assert_eq!(
5632            std::fs::metadata(&retained).unwrap().len(),
5633            64 * 1024 * 1024
5634        );
5635        assert!(
5636            blocked
5637                .retained
5638                .iter()
5639                .any(|r| r.contains("per-run storage limit")),
5640            "{blocked:?}"
5641        );
5642    }
5643    #[tokio::test]
5644    async fn source_eviction_migrates_historical_records_without_rewriting_them() {
5645        let f = Fx::new().await;
5646        let mut state = inactive_source(&f, "ec14", true).await;
5647        historical_review(&mut state, &f.base);
5648        historical_review(&mut state, &f.base);
5649        state.status = RunStatus::Superseded;
5650        state.save_under(&f.home).unwrap();
5651        let review = reviewer(&f, &state, &f.base).await;
5652        let task = f.task(std::slice::from_ref(&state));
5653        let path = f.home.join("runs").join(&state.id).join("run.json");
5654        let mut raw: serde_json::Value =
5655            serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5656        raw["schema"] = 10.into();
5657        raw.as_object_mut().unwrap().remove("output_activity_at");
5658        // Historical writers persisted null/empty defaults that current serde
5659        // omits. These are known fields, not unknown recovery authorities.
5660        raw["resource_wait"] = serde_json::Value::Null;
5661        raw["resource_block"] = serde_json::Value::Null;
5662        raw["driver_claimed"] = serde_json::Value::Null;
5663        raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
5664        // Actual legacy spelling accepted by Graph's serde alias.
5665        let graph = raw["config"]["graph"].as_object_mut().unwrap();
5666        let count = graph.remove("implementers").unwrap();
5667        graph.insert("candidates".into(), count);
5668        let queue = crate::queue::Queue::at(f.home.join("queue"));
5669        let task_path = queue.path_of(&task.id);
5670        let mut task_raw: serde_json::Value =
5671            serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
5672        task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
5673        let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
5674        std::fs::write(&task_path, &task_historical).unwrap();
5675        let historical = serde_json::to_vec_pretty(&raw).unwrap();
5676        std::fs::write(&path, &historical).unwrap();
5677        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5678        assert_eq!(report.evicted_copies, 1, "{report:?}");
5679        assert!(!review.exists());
5680        assert_eq!(std::fs::read(&path).unwrap(), historical);
5681        assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
5682        assert!(state.candidates[0].worktree.exists());
5683        assert_eq!(
5684            crate::queue::Queue::at(f.home.join("queue"))
5685                .get(&task.id)
5686                .unwrap()
5687                .runs,
5688            task.runs
5689        );
5690    }
5691    #[tokio::test]
5692    async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
5693        let f = Fx::new().await;
5694        let state = inactive_source(&f, "ed01", true).await;
5695        let review = reviewer(&f, &state, &f.base).await;
5696        std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
5697        git::git(&review, &["add", "."]).await.unwrap();
5698        let tree = git::git(&review, &["write-tree"]).await.unwrap();
5699        let tip = git::git(
5700            &review,
5701            &[
5702                "commit-tree",
5703                &tree,
5704                "-p",
5705                &f.base,
5706                "-m",
5707                "temporary detached source",
5708            ],
5709        )
5710        .await
5711        .unwrap();
5712        git::git(&review, &["checkout", "--detach", &tip])
5713            .await
5714            .unwrap();
5715        git::git(&review, &["checkout", "--detach", &f.base])
5716            .await
5717            .unwrap();
5718        git::git(&review, &["reflog", "delete", "HEAD@{1}"])
5719            .await
5720            .unwrap();
5721        git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
5722            .await
5723            .unwrap();
5724        let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
5725            .await
5726            .unwrap();
5727        assert!(!shown.lines().any(|oid| oid == tip));
5728        let admin = PathBuf::from(
5729            git::git(&review, &["rev-parse", "--absolute-git-dir"])
5730                .await
5731                .unwrap(),
5732        );
5733        let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
5734        assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
5735        fn interrupt(_: &Path) -> Result<()> {
5736            anyhow::bail!("fixture interruption before unlink")
5737        }
5738        let mut pending = Report::default();
5739        copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
5740        assert!(review.exists());
5741        let ledger = f
5742            .home
5743            .join("runs")
5744            .join(&state.id)
5745            .join("artifacts/eviction-review-1.json");
5746        let complete = std::fs::read(&ledger).unwrap();
5747        let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
5748        incomplete["history"]
5749            .as_object_mut()
5750            .unwrap()
5751            .remove(&tip)
5752            .unwrap();
5753        std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
5754        let mut blocked = Report::default();
5755        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
5756        assert!(review.exists());
5757        assert!(
5758            blocked
5759                .retained
5760                .iter()
5761                .any(|reason| reason.contains("incomplete reflog checkpoint")),
5762            "{blocked:?}"
5763        );
5764        std::fs::write(&ledger, complete).unwrap();
5765        let mut report = Report::default();
5766        copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5767        assert_eq!(report.evicted_copies, 1, "{report:?}");
5768        git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5769            .await
5770            .unwrap();
5771        git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5772        assert!(
5773            !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
5774                .await
5775                .unwrap()
5776                .ok()
5777        );
5778        std::fs::remove_dir_all(&f.repo).unwrap();
5779        let destination = f._temp.path().join("restore-old-only");
5780        restore_seat(&f.home, &state.id, &tip, &destination)
5781            .await
5782            .unwrap();
5783        assert_eq!(
5784            std::fs::read(destination.join("old-only")).unwrap(),
5785            b"old-only\0binary\xff"
5786        );
5787    }
5788
5789    #[tokio::test]
5790    async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
5791        fn interrupt(_: &Path) -> Result<()> {
5792            anyhow::bail!("fixture interruption before unlink")
5793        }
5794        fn vanish(path: &Path) -> Result<()> {
5795            std::fs::remove_dir_all(path)?;
5796            anyhow::bail!("fixture interruption before unregistration")
5797        }
5798        for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
5799            let f = Fx::new().await;
5800            let mut state = inactive_source(&f, "ed02", true).await;
5801            state.driver_exited = false;
5802            state.driver_claimed = Some(true);
5803            state.driver_pid = Some(std::process::id());
5804            state.driver_started_at = crate::proc::process_started_at(std::process::id());
5805            state.begin_pass();
5806            state.status = RunStatus::Superseded;
5807            state.save_under(&f.home).unwrap();
5808            let review = reviewer(&f, &state, &f.base).await;
5809            // The same matching identity written by DriverGuard::record, left
5810            // behind when a process exits without its cleanup guard running.
5811            let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
5812                "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
5813            std::fs::write(
5814                f.home.join("runs").join(&state.id).join("driver.json"),
5815                serde_json::to_vec(&identity).unwrap(),
5816            )
5817            .unwrap();
5818            let live = if stage == "live-driver" {
5819                Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
5820            } else {
5821                None
5822            };
5823            assert_eq!(
5824                state.liveness_under(&f.home, false),
5825                if live.is_some() {
5826                    Liveness::Live
5827                } else {
5828                    Liveness::Dead
5829                }
5830            );
5831            let mut report = Report::default();
5832            if stage == "intent" || stage == "missing-registration" {
5833                copies::evict_with(
5834                    &f.home,
5835                    &Fx::disk(),
5836                    Fx::now(),
5837                    &mut report,
5838                    if stage == "intent" { interrupt } else { vanish },
5839                )
5840                .await;
5841                assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
5842                let mut retry = Report::default();
5843                copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5844                assert!(!review.exists(), "{stage}: {retry:?}");
5845                let ledger = f
5846                    .home
5847                    .join("runs")
5848                    .join(&state.id)
5849                    .join("artifacts/eviction-review-1.json");
5850                let receipt: serde_json::Value =
5851                    serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
5852                assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
5853            } else {
5854                copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5855                assert_eq!(
5856                    report.evicted_copies,
5857                    usize::from(live.is_none()),
5858                    "{stage}: {report:?}"
5859                );
5860                assert_eq!(review.exists(), live.is_some());
5861            }
5862            drop(live);
5863            assert!(
5864                !RunState::load_under(&state.id, &f.home)
5865                    .unwrap()
5866                    .driver_exited
5867            );
5868        }
5869    }
5870}