Skip to main content

magi/
retention.rs

1//! Bounded reclamation of retained attempts and Cargo outputs. Source recovery
2//! and build eviction have separate authority: evicting a target never retires a run.
3pub mod reconcile;
4
5use crate::{
6    git,
7    proc::Quiet as _,
8    run::{Liveness, RunState},
9};
10use anyhow::{Context, Result, ensure};
11use jiff::Timestamp;
12use serde::{Deserialize, Serialize};
13use std::{
14    fs::{File, OpenOptions},
15    path::{Path, PathBuf},
16};
17
18/// An OS lease on a worktree's outputs, independent of repository coordination.
19/// Kept by the process cleanup owner until descendants terminate, including
20/// reparented tools retaining the inherited liveness witness.
21#[derive(Debug)]
22pub(crate) struct OutputLease {
23    _file: File,
24}
25
26impl OutputLease {
27    #[cfg(unix)]
28    pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
29        use std::os::fd::AsRawFd;
30        self._file.as_raw_fd()
31    }
32
33    pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
34        if !worktree.join(".git").is_file() {
35            return Ok(None);
36        }
37        Self::take(worktree, false).map(Some)
38    }
39    pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
40        use std::hash::{Hash, Hasher};
41        let mut hash = std::collections::hash_map::DefaultHasher::new();
42        let mut normalized = cache.to_owned();
43        let mut missing = Vec::new();
44        while !normalized.exists() {
45            let Some(name) = normalized.file_name() else {
46                break;
47            };
48            missing.push(name.to_owned());
49            if !normalized.pop() {
50                break;
51            }
52        }
53        normalized = normalized.canonicalize().unwrap_or(normalized);
54        for part in missing.into_iter().rev() {
55            normalized.push(part);
56        }
57        normalized.hash(&mut hash);
58        let dir = home.join("cache-leases");
59        std::fs::create_dir_all(&dir)?;
60        Self::lock(
61            &dir.join(format!("outputs-{:x}.lock", hash.finish())),
62            exclusive,
63        )
64    }
65    pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
66        let out = std::process::Command::new("git")
67            .args(["rev-parse", "--absolute-git-dir"])
68            .current_dir(worktree)
69            .quiet()
70            .output()?;
71        ensure!(
72            out.status.success(),
73            "cannot identify output lease directory"
74        );
75        let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
76        Self::lock(&dir.join("magi-output.lock"), exclusive)
77    }
78    fn lock(path: &Path, exclusive: bool) -> Result<Self> {
79        let file = OpenOptions::new()
80            .create(true)
81            .truncate(false)
82            .read(true)
83            .write(true)
84            .open(path)?;
85        #[cfg(unix)]
86        {
87            use std::os::fd::AsRawFd;
88            let mode = if exclusive {
89                libc::LOCK_EX
90            } else {
91                libc::LOCK_SH
92            };
93            // SAFETY: live file descriptor, no retained pointers.
94            ensure!(
95                unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
96                "worktree outputs are in use or being reclaimed"
97            );
98        }
99        #[cfg(not(unix))]
100        let _ = exclusive;
101        Ok(Self { _file: file })
102    }
103}
104#[cfg(unix)]
105impl Drop for OutputLease {
106    fn drop(&mut self) {
107        use std::os::fd::AsRawFd;
108        // SAFETY: unlock this guard's live descriptor, including inherited copies.
109        unsafe {
110            libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
111        }
112    }
113}
114
115#[derive(Default, Debug, Serialize, Deserialize)]
116#[serde(default)]
117pub(crate) struct Report {
118    pub checked_at: Option<Timestamp>,
119    pub limit_bytes: u64,
120    pub reclaim_budget_bytes: u64,
121    pub measured_bytes: u64,
122    pub freed_bytes: u64,
123    pub retired: usize,
124    pub retained: Vec<String>,
125    pub failures: Vec<String>,
126}
127
128fn plain_dir(path: &Path) -> bool {
129    std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
130}
131
132/// Old run records outlive their checkout and output directories. Only an
133/// absent path means nothing to reclaim; inaccessible paths remain errors.
134fn run_paths_missing(state: &RunState) -> Result<bool> {
135    for path in [state.worktree_root(), state.repo.clone()] {
136        match std::fs::metadata(&path) {
137            Ok(_) => {}
138            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
139            Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
140        }
141    }
142    Ok(false)
143}
144
145/// Enumerate actual registered worktrees, not just candidates in run.json.
146async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
147    let root = state.worktree_root();
148    if !plain_dir(&root) {
149        return Ok(Vec::new());
150    }
151    let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
152    registered_worktrees(state, &list)
153}
154
155fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
156    let root = state.worktree_root();
157    if !plain_dir(&root) {
158        return Ok(Vec::new());
159    }
160    let root = root.canonicalize()?;
161    let mut paths = Vec::new();
162    for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
163        let path = PathBuf::from(path);
164        if plain_dir(&path)
165            && path.parent() == Some(root.as_path())
166            && path.canonicalize()? == path
167            && path.join(".git").is_file()
168        {
169            paths.push(path);
170        }
171    }
172    Ok(paths)
173}
174
175/// Recognize Cargo's generated directory, never merely an ignored directory.
176/// Unknown root entries and symlinked cache identity/profile paths fail closed.
177async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
178    let target = worktree.join("target");
179    ensure!(plain_dir(&target), "target absent or not a plain directory");
180    ensure!(target.canonicalize()? == target, "target path changed");
181    ensure!(
182        git::git(worktree, &["ls-files", "--", "target"])
183            .await?
184            .is_empty(),
185        "tracked target contents"
186    );
187    ensure!(
188        git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
189            .await?
190            .ok(),
191        "target is not ignored"
192    );
193    for name in ["CACHEDIR.TAG", ".rustc_info.json"] {
194        let path = target.join(name);
195        ensure!(
196            std::fs::symlink_metadata(&path)?.is_file() && path.canonicalize()? == path,
197            "cache identity is not a plain file"
198        );
199    }
200    let tag = std::fs::read_to_string(target.join("CACHEDIR.TAG"))?;
201    ensure!(
202        tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55d"),
203        "unknown Cargo cache tag"
204    );
205    let info: serde_json::Value =
206        serde_json::from_slice(&std::fs::read(target.join(".rustc_info.json"))?)?;
207    ensure!(
208        info.get("rustc_fingerprint").is_some(),
209        "unknown Cargo identity"
210    );
211    for entry in std::fs::read_dir(&target)? {
212        let entry = entry?;
213        let name = entry.file_name();
214        let name = name.to_str().context("non-UTF8 output")?;
215        ensure!(
216            matches!(
217                name,
218                "debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json"
219            ),
220            "unrecognized target entry {name}"
221        );
222        if matches!(name, "debug" | "release") {
223            ensure!(
224                plain_dir(&entry.path()) && entry.path().canonicalize()? == entry.path(),
225                "profile is not a plain directory"
226            );
227        }
228    }
229    Ok(target)
230}
231
232fn artifact_signature(path: &Path) -> Result<bool> {
233    use std::io::Read as _;
234    let mut header = [0; 8];
235    let n = File::open(path)?.read(&mut header)?;
236    let header = &header[..n];
237    Ok(header.starts_with(b"\x7fELF")
238        || header.starts_with(b"!<arch>\n")
239        || header.starts_with(b"rust\0\0\0")
240        || header.starts_with(b"MZ\x90\0")
241        || [
242            [0xfe, 0xed, 0xfa, 0xce],
243            [0xce, 0xfa, 0xed, 0xfe],
244            [0xfe, 0xed, 0xfa, 0xcf],
245            [0xcf, 0xfa, 0xed, 0xfe],
246            [0xca, 0xfe, 0xba, 0xbe],
247            [0xca, 0xfe, 0xba, 0xbf],
248        ]
249        .iter()
250        .any(|magic| header.starts_with(magic)))
251}
252
253/// Legacy tags identify a cache, not ownership of every file inside it.
254/// Only direct rustc outputs named by dep-info may be reclaimed. Build-script
255/// output, incremental caches, unknown files and the manifests themselves stay.
256fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
257    let mut outputs = std::collections::BTreeSet::new();
258    for profile in ["debug", "release"] {
259        for dir in [target.join(profile), target.join(profile).join("deps")] {
260            if !plain_dir(&dir) || dir.canonicalize()? != dir {
261                continue;
262            }
263            for entry in std::fs::read_dir(&dir)? {
264                let path = entry?.path();
265                if path.extension().is_none_or(|ext| ext != "d")
266                    || !std::fs::symlink_metadata(&path)?.is_file()
267                    || std::fs::metadata(&path)?.len() > 1024 * 1024
268                    || path.canonicalize()? != path
269                {
270                    continue;
271                }
272                let body = std::fs::read_to_string(&path)?;
273                for line in body.lines() {
274                    let Some((names, dependencies)) = line.split_once(": ") else {
275                        continue;
276                    };
277                    if dependencies.trim().is_empty() {
278                        continue;
279                    }
280                    // Rustc escapes spaces and backslashes in Make paths.
281                    // Unsupported Make syntax fails closed for that record.
282                    let mut words = Vec::new();
283                    let mut word = String::new();
284                    let mut escaped = false;
285                    for c in names.chars() {
286                        if escaped {
287                            word.push(c);
288                            escaped = false;
289                        } else if c == '\\' {
290                            escaped = true;
291                        } else if c.is_whitespace() {
292                            if !word.is_empty() {
293                                words.push(std::mem::take(&mut word));
294                            }
295                        } else {
296                            word.push(c);
297                        }
298                    }
299                    if escaped || word.contains(['$', '%']) {
300                        continue;
301                    }
302                    if !word.is_empty() {
303                        words.push(word);
304                    }
305                    for name in words {
306                        let output = PathBuf::from(name);
307                        let stem = path.file_stem().unwrap_or_default().to_string_lossy();
308                        let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
309                        let artifact_name =
310                            output_stem == stem || output_stem == format!("lib{stem}");
311                        let artifact_kind = output.extension().is_none_or(|ext| {
312                            matches!(
313                                ext.to_str(),
314                                Some(
315                                    "rlib"
316                                        | "rmeta"
317                                        | "so"
318                                        | "dylib"
319                                        | "a"
320                                        | "lib"
321                                        | "dll"
322                                        | "exe"
323                                        | "pdb"
324                                )
325                            )
326                        });
327                        if artifact_name
328                            && artifact_kind
329                            && output.parent() == Some(dir.as_path())
330                            && output.extension().is_none_or(|ext| ext != "d")
331                            && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
332                            && output.canonicalize().is_ok_and(|p| p == output)
333                            && artifact_signature(&output)?
334                        {
335                            outputs.insert(output);
336                        }
337                    }
338                }
339            }
340        }
341    }
342    Ok(outputs.into_iter().collect())
343}
344
345#[derive(Serialize, Deserialize)]
346struct CargoResidue {
347    target: PathBuf,
348    files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
349}
350
351// Keep the exact metadata as historical evidence. An old proof cannot adopt
352// newly added files, changed metadata, symlinks or a different worktree.
353fn residue_files(
354    target: &Path,
355    outputs: &[PathBuf],
356) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
357    let mut files = std::collections::BTreeMap::new();
358    let mut dirs = vec![target.to_owned()];
359    let mut bytes = 0usize;
360    let mut entries = 0usize;
361    while let Some(dir) = dirs.pop() {
362        for entry in std::fs::read_dir(dir)? {
363            let path = entry?.path();
364            entries += 1;
365            ensure!(entries <= 4096, "residue inventory too large");
366            let metadata = std::fs::symlink_metadata(&path)?;
367            ensure!(
368                !metadata.file_type().is_symlink(),
369                "symlink in Cargo residue"
370            );
371            if metadata.is_dir() {
372                dirs.push(path);
373            } else {
374                ensure!(metadata.is_file(), "non-file in Cargo residue");
375                if outputs.contains(&path) {
376                    continue;
377                }
378                ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
379                let body = std::fs::read(&path)?;
380                bytes = bytes.saturating_add(body.len());
381                ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
382                files.insert(path.strip_prefix(target)?.to_owned(), body);
383            }
384        }
385    }
386    Ok(files)
387}
388
389fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
390    Ok(home
391        .join("runs")
392        .join(id)
393        .join("build-residue")
394        .join(worktree.file_name().context("missing worktree name")?))
395}
396
397fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
398    let files = residue_files(target, outputs)?;
399    for (relative, body) in &files {
400        let path = target.join(relative);
401        let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
402        let lock = matches!(
403            relative.to_str(),
404            Some("debug/.cargo-lock" | "release/.cargo-lock")
405        ) && body.is_empty();
406        let dep_info = path.extension().is_some_and(|ext| ext == "d")
407            && outputs.iter().any(|output| {
408                output.parent() == path.parent()
409                    && (output.file_stem() == path.file_stem()
410                        || output.file_stem().is_some_and(|stem| {
411                            stem.to_string_lossy()
412                                == format!(
413                                    "lib{}",
414                                    path.file_stem().unwrap_or_default().to_string_lossy()
415                                )
416                        }))
417            });
418        ensure!(identity || lock || dep_info, "unproven Cargo residue");
419    }
420    let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
421    std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
422    let tmp = proof.with_extension("tmp");
423    std::fs::write(
424        &tmp,
425        serde_json::to_vec(&CargoResidue {
426            target: target.to_owned(),
427            files,
428        })?,
429    )?;
430    std::fs::rename(tmp, proof)?;
431    Ok(())
432}
433
434pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
435    let target = cargo_target(worktree).await?;
436    let locks = cargo_locks(&target)?;
437    let proof: CargoResidue =
438        serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
439    ensure!(
440        proof.target == target && proof.files == residue_files(&target, &[])?,
441        "Cargo residue changed or contains unproven files"
442    );
443    Ok(locks)
444}
445
446/// Supplemental exclusion for Cargo builds launched outside the managed seats.
447pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
448    let mut held = Vec::new();
449    for profile in ["debug", "release"] {
450        let lock = target.join(profile).join(".cargo-lock");
451        if lock.exists() {
452            ensure!(
453                std::fs::symlink_metadata(&lock)?.is_file() && lock.canonicalize()? == lock,
454                "Cargo lock is not a plain file"
455            );
456            let file = OpenOptions::new().read(true).write(true).open(lock)?;
457            crate::pr_head::try_lock_file(&file).context("Cargo build is active")?;
458            held.push(file);
459        }
460    }
461    Ok(held)
462}
463
464/// All repositories share one cap and one per-pass deletion budget. The report
465/// survives a restart and explains every skipped run/target or failed operation.
466pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
467    let home = home.to_owned();
468    let disk = disk.clone();
469    let runtime = tokio::runtime::Handle::current();
470    tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
471        .await
472        .unwrap_or_else(|e| Report {
473            failures: vec![format!("retention worker failed: {e}")],
474            ..Report::default()
475        })
476}
477
478#[derive(Default, Serialize, Deserialize)]
479struct Measurement {
480    at: Option<Timestamp>,
481    bytes: u64,
482}
483
484async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
485    let mut report = Report {
486        checked_at: Some(now),
487        limit_bytes: disk.worktree_limit_bytes,
488        reclaim_budget_bytes: disk.worktree_reclaim_bytes,
489        ..Report::default()
490    };
491    if !cfg!(unix) {
492        report
493            .failures
494            .push("Automatic output reclamation requires Unix process containment".into());
495        return report;
496    }
497    let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
498        return report;
499    };
500    let mut targets = Vec::new();
501    let mut registrations = std::collections::HashMap::new();
502    let measurements_path = home.join("output-measurements.json");
503    let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
504        std::fs::read(&measurements_path)
505            .ok()
506            .and_then(|b| serde_json::from_slice(&b).ok())
507            .unwrap_or_default();
508    let mut observed = std::collections::BTreeSet::new();
509    let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
510        .into_iter()
511        .flatten()
512        .flatten()
513        .map(|e| e.file_name().to_string_lossy().into_owned())
514        .collect();
515    ids.sort();
516    for id in ids {
517        if !crate::run::is_run_id(&id) {
518            continue;
519        }
520        let state = match RunState::load_under(&id, home) {
521            Ok(state) => state,
522            Err(e) => {
523                report
524                    .retained
525                    .push(format!("{id}: unreadable run ownership: {e:#}"));
526                continue;
527            }
528        };
529        if disk.worktree_limit_bytes == 0 {
530            continue;
531        }
532        match run_paths_missing(&state) {
533            Ok(true) => continue,
534            Ok(false) => {}
535            Err(e) => {
536                report.failures.push(format!("{id}: {e:#}"));
537                continue;
538            }
539        }
540        let list = match registrations.entry(state.repo.clone()) {
541            std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
542            std::collections::hash_map::Entry::Vacant(entry) => {
543                match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
544                    Ok(list) => entry.insert(list),
545                    Err(e) => {
546                        // The checkout may disappear after the inventory check.
547                        // Do not confuse a missing Git executable with that case.
548                        if matches!(run_paths_missing(&state), Ok(true)) {
549                            continue;
550                        }
551                        report.failures.push(format!("{id}: {e:#}"));
552                        continue;
553                    }
554                }
555            }
556        };
557        let paths = match registered_worktrees(&state, list) {
558            Ok(paths) => paths,
559            Err(e) => {
560                report.failures.push(format!("{id}: {e:#}"));
561                continue;
562            }
563        };
564        for path in paths {
565            if !path.join("target").exists() {
566                continue;
567            }
568            let target = path.join("target");
569            observed.insert(target.clone());
570            let pressure =
571                crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
572            let measurement = measurements.entry(target).or_default();
573            if pressure
574                || measurement
575                    .at
576                    .is_none_or(|at| crate::clean::due(now, at, 300))
577            {
578                measurement.bytes = crate::disk::dir_size(&path.join("target"));
579                measurement.at = Some(now);
580            }
581            let size = measurement.bytes;
582            report.measured_bytes = report.measured_bytes.saturating_add(size);
583            targets.push((state.updated_at, id.clone(), path, size));
584        }
585    }
586    targets.sort_by_key(|t| t.0);
587    let mut remaining = report.measured_bytes;
588    for (updated, id, path, size) in targets {
589        if disk.worktree_limit_bytes == 0
590            || remaining <= disk.worktree_limit_bytes
591                && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
592        {
593            report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
594            continue;
595        }
596        if report.freed_bytes.saturating_add(size) > disk.worktree_reclaim_bytes {
597            report.retained.push(format!(
598                "{id}: target exceeds remaining per-pass byte budget"
599            ));
600            continue;
601        }
602        let result = async {
603            ensure!(
604                crate::clean::due(now, updated, disk.fold_grace_secs),
605                "within grace period"
606            );
607            let _owner = crate::pr_head::owner_lock(home, &id)?;
608            let state = RunState::load_under(&id, home)?;
609            ensure!(
610                state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
611                    == Liveness::Dead,
612                "run live or ownership unknown"
613            );
614            ensure!(crate::clean::due(now, state.updated_at, disk.fold_grace_secs), "recently used after inventory");
615            let _driver = crate::driver::maintenance(home, &id)?;
616            let _outputs = OutputLease::take(&path, true)?;
617            ensure!(
618                worktrees(&state).await?.contains(&path),
619                "worktree registration changed"
620            );
621            let target = cargo_target(&path).await?;
622            let _cargo = cargo_locks(&target)?;
623            let outputs = declared_outputs(&target)?;
624            ensure!(!outputs.is_empty(), "no dep-info-proven outputs; unknown files retained");
625            let current_size = outputs.iter().try_fold(0u64, |bytes, p| {
626                Ok::<_, std::io::Error>(bytes.saturating_add(std::fs::metadata(p)?.len()))
627            })?;
628            ensure!(report.freed_bytes.saturating_add(current_size) <= disk.worktree_reclaim_bytes,
629                "target grew beyond remaining per-pass budget");
630            // Record adoption before deletion; interrupted deletion is revalidated
631            // on the next pass. No source file or branch is modified.
632            let ledger = home.join("runs").join(&id).join("build-outputs.json");
633            std::fs::write(
634                ledger,
635                serde_json::to_vec(
636                    &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","files":outputs}),
637                )?,
638            )?;
639            // Unknown leftovers still permit artifact-only eviction, but never
640            // authorize retirement. Only a complete proven residue is adopted.
641            if let Err(error) = record_residue(home, &id, &target, &outputs) {
642                report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
643            }
644            for output in &outputs {
645                std::fs::remove_file(output)?;
646            }
647            measurements.remove(&target);
648            report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
649            Ok::<_, anyhow::Error>(current_size)
650        }
651        .await;
652        match result {
653            Ok(freed) => {
654                report.freed_bytes = report.freed_bytes.saturating_add(freed);
655                remaining = remaining.saturating_sub(freed);
656            }
657            Err(e) => report.retained.push(format!(
658                "{id}/{}: {e:#}",
659                path.file_name().unwrap_or_default().to_string_lossy()
660            )),
661        }
662    }
663    measurements.retain(|path, _| observed.contains(path));
664    if let Ok(body) = serde_json::to_vec(&measurements)
665        && let Err(e) = std::fs::write(&measurements_path, body)
666    {
667        report.failures.push(format!("output measurements: {e}"));
668    }
669    retire_empty(home, disk, now, &mut report).await;
670    if !report.failures.is_empty()
671        || disk.worktree_limit_bytes > 0
672            && remaining > disk.worktree_limit_bytes
673            && report.freed_bytes == 0
674    {
675        crate::notices::raise_in(
676            home,
677            crate::notices::Notice::warn(
678                "disk:retention",
679                "Retained build outputs could not be brought under the disk cap. Inspect the cleanup report for active owners, retained recovery sources and failures.",
680            ),
681        );
682    }
683    let store = home.join("retention.json");
684    if let Ok(body) = serde_json::to_vec_pretty(&report) {
685        let tmp = home.join("retention.json.tmp");
686        if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
687            tracing::warn!("retention report could not be saved: {e}");
688        }
689    }
690    tracing::info!(
691        measured_bytes = report.measured_bytes,
692        freed_bytes = report.freed_bytes,
693        retired = report.retired,
694        retained = report.retained.len(),
695        failures = report.failures.len(),
696        "retained worktree cleanup"
697    );
698    for reason in report.retained.iter().chain(&report.failures) {
699        tracing::debug!(%reason, "worktree retained");
700    }
701    report
702}
703
704async fn retire_empty(
705    home: &Path,
706    disk: &crate::config::Disk,
707    now: Timestamp,
708    report: &mut Report,
709) {
710    if !disk.auto_fold {
711        return;
712    }
713    let queue = crate::queue::Queue::at(home.join("queue"));
714    let questions = crate::ask::Questions::at(home.join("questions"));
715    let open = questions.list();
716    for listed in queue.list() {
717        let Ok(_claim) = queue.claim(&listed.id) else {
718            continue;
719        };
720        let Ok(task) = queue.get(&listed.id) else {
721            continue;
722        };
723        if open
724            .iter()
725            .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
726        {
727            report.retained.push(format!(
728                "{}: task has an unanswered owner decision",
729                task.id
730            ));
731            continue;
732        }
733        let sources: Vec<_> = task
734            .runs
735            .iter()
736            .collect::<std::collections::BTreeSet<_>>()
737            .into_iter()
738            .cloned()
739            .collect();
740        let mut useful = Vec::new();
741        let mut selected = Vec::new();
742        let mut public = Vec::new();
743        let mut unreadable = false;
744        for id in &sources {
745            let Ok(state) = RunState::load_under(id, home) else {
746                unreadable = true;
747                continue;
748            };
749            if state.released() || state.retired_to.is_some() {
750                continue;
751            }
752            if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
753                selected.push(state.clone());
754            }
755            if state.pr.is_some() && state.pr_taken_over_by.is_none() {
756                public.push(state.clone());
757            }
758            if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
759                useful.push(state);
760            }
761        }
762        // Prefer a durable owner-approved selection, then one existing PR
763        // owner, then an unambiguous useful source. Distinct historical changes
764        // without an owner selection cannot be resolved by creation time.
765        let pool = if !selected.is_empty() {
766            selected
767        } else if !public.is_empty() {
768            public
769        } else {
770            useful
771        };
772        if unreadable || pool.len() != 1 {
773            report.retained.push(format!(
774                "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
775                task.id,
776                sources.len()
777            ));
778            continue;
779        }
780        let authority = &pool[0];
781        if authority.candidates.is_empty() {
782            continue;
783        }
784        let latest = &authority.id;
785        for id in &sources {
786            if id == latest || report.retired >= 5 {
787                continue;
788            }
789            if let Ok(state) = RunState::load_under(id, home)
790                && state.retired_to.as_ref() == Some(latest)
791                && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
792            {
793                continue;
794            }
795            let result = async {
796                ensure!(
797                    !task
798                        .resume_override
799                        .as_ref()
800                        .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
801                    "operator-pinned recovery run"
802                );
803                ensure!(
804                    !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
805                    "selected replacement recovery source"
806                );
807                let _owner = crate::pr_head::owner_lock(home, id)?;
808                let mut state = RunState::load_under(id, home)?;
809                ensure!(
810                    state.repo == authority.repo,
811                    "repository differs from recovery authority"
812                );
813                ensure!(
814                    !crate::pr_recovery::explicit_sources_retained(home, id),
815                    "explicitly retained recovery source"
816                );
817                ensure!(
818                    state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
819                        == Liveness::Dead,
820                    "run live or ownership unknown"
821                );
822                let _driver = crate::driver::maintenance(home, id)?;
823                ensure!(
824                    crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
825                    "within grace period"
826                );
827                ensure!(
828                    !open.iter().any(|q| q.run == *id && q.status.open()),
829                    "unanswered owner decision"
830                );
831                ensure!(
832                    state.pr.is_none()
833                        && state.pr_head.is_none()
834                        && state.pr_recovery.is_none()
835                        && state.land_approval.is_none()
836                        && state.merge_decision.is_none()
837                        && state.reviews.is_empty()
838                        && state.recovery.is_none()
839                        && state.candidate_preservation.is_empty()
840                        && state.interrupted_implementations.is_empty()
841                        && !state.released(),
842                    "PR ownership or recovery checkpoint"
843                );
844                let proof = prove_empty(home, &state, latest).await?;
845                commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
846                // Durable retirement precedes removal. A crash retries removal
847                // using the same proof; it cannot make this attempt resumable.
848                for path in &proof.paths {
849                    ensure!(
850                        git::worktree_remove_clean(&state.repo, path).await?,
851                        "Git refused clean worktree removal"
852                    );
853                }
854                Ok::<_, anyhow::Error>(())
855            }
856            .await;
857            match result {
858                Ok(()) => report.retired += 1,
859                Err(e) => report.retained.push(format!("{id}: {e:#}")),
860            }
861        }
862    }
863}
864
865struct EmptyProof {
866    paths: Vec<PathBuf>,
867    _outputs: Vec<OutputLease>,
868    _cargo: Vec<File>,
869}
870
871// Compare actual source bytes with the original tree independently of index
872// stat caches, assume-unchanged flags, sparse checkout and fsmonitor hints.
873async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
874    let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
875    let mut files = Vec::new();
876    let mut blobs = Vec::new();
877    for record in tree.split('\0').filter(|s| !s.is_empty()) {
878        let (header, name) = record.split_once('\t').context("unknown tree record")?;
879        let parts: Vec<_> = header.split_whitespace().collect();
880        ensure!(
881            parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
882            "unsupported tracked source kind"
883        );
884        let path = worktree.join(name);
885        let meta = std::fs::symlink_metadata(&path)?;
886        ensure!(
887            meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
888            "noncanonical tracked source"
889        );
890        #[cfg(unix)]
891        {
892            use std::os::unix::fs::PermissionsExt as _;
893            ensure!(
894                (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
895                "tracked executable mode differs from original base"
896            );
897        }
898        files.push(path.to_str().context("non UTF-8 source")?.to_owned());
899        blobs.push(parts[2].to_owned());
900    }
901    for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
902        let mut args = vec!["hash-object", "--no-filters", "--"];
903        args.extend(paths.iter().map(String::as_str));
904        let actual = git::git(worktree, &args).await?;
905        ensure!(
906            actual.lines().eq(expected.iter().map(String::as_str)),
907            "tracked bytes differ from original base"
908        );
909    }
910    Ok(())
911}
912
913async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
914    let id = &state.id;
915    let paths = worktrees(state).await?;
916    let mut guards = Vec::new();
917    let mut cargo_guards = Vec::new();
918    for path in &paths {
919        guards.push(OutputLease::take(path, true)?);
920        ensure!(
921            git::rev_parse(path, "HEAD").await? == state.base_commit,
922            "unique or unrecorded commits"
923        );
924        ensure!(
925            git::git(
926                path,
927                &[
928                    "--no-optional-locks",
929                    "status",
930                    "--porcelain",
931                    "--untracked-files=all"
932                ]
933            )
934            .await?
935            .is_empty(),
936            "tracked or untracked source changes"
937        );
938        ensure!(
939            git::git(
940                path,
941                &[
942                    "--no-optional-locks",
943                    "status",
944                    "--porcelain",
945                    "--untracked-files=all",
946                    "--ignored",
947                    "--",
948                    ".",
949                    ":(exclude)target"
950                ]
951            )
952            .await?
953            .is_empty(),
954            "source changes, ignored files or build outputs"
955        );
956        tracked_base(path, &state.base_commit).await?;
957        match std::fs::symlink_metadata(path.join("target")) {
958            Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
959            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
960            Err(error) => return Err(error.into()),
961        }
962        for checkpoint in [
963            "rebase-merge",
964            "rebase-apply",
965            "sequencer",
966            "BISECT_LOG",
967            "REVERT_HEAD",
968            "AUTO_MERGE",
969            "MERGE_MSG",
970            "CHERRY_PICK_HEAD",
971            "MERGE_HEAD",
972            "REBASE_HEAD",
973        ] {
974            let location = git::git(
975                path,
976                &[
977                    "rev-parse",
978                    "--path-format=absolute",
979                    "--git-path",
980                    checkpoint,
981                ],
982            )
983            .await?;
984            ensure!(
985                matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
986                "retained Git recovery metadata: {checkpoint}"
987            );
988        }
989        // Git also initializes ORIG_HEAD when creating a worktree.
990        // Only an origin outside the base is a recovery source.
991        let origin_path = git::git(
992            path,
993            &[
994                "rev-parse",
995                "--path-format=absolute",
996                "--git-path",
997                "ORIG_HEAD",
998            ],
999        )
1000        .await?;
1001        match std::fs::symlink_metadata(origin_path.trim()) {
1002            Ok(meta) => {
1003                ensure!(
1004                    meta.is_file() && !meta.file_type().is_symlink(),
1005                    "unknown ORIG_HEAD artifact"
1006                );
1007                git::rev_parse(path, "ORIG_HEAD")
1008                    .await
1009                    .context("unreadable ORIG_HEAD")?;
1010            }
1011            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1012            Err(e) => return Err(e.into()),
1013        }
1014        if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1015            ensure!(
1016                git::git_raw(
1017                    path,
1018                    &["merge-base", "--is-ancestor", &original, &state.base_commit]
1019                )
1020                .await?
1021                .ok(),
1022                "ORIG_HEAD retains commits outside the original base"
1023            );
1024        }
1025        let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1026        for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1027            ensure!(
1028                git::git_raw(
1029                    path,
1030                    &["merge-base", "--is-ancestor", head, &state.base_commit]
1031                )
1032                .await?
1033                .ok(),
1034                "worktree reflog retains commits outside the original base"
1035            );
1036        }
1037        for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1038            ensure!(
1039                !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1040                    .await?
1041                    .ok(),
1042                "unfinished Git operation"
1043            );
1044        }
1045    }
1046    // Missing worktrees cannot establish that unrecorded prep edits
1047    // were empty. Recorded candidates must also point at the base.
1048    if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1049        return Ok(EmptyProof {
1050            paths,
1051            _outputs: guards,
1052            _cargo: cargo_guards,
1053        });
1054    }
1055    ensure!(!paths.is_empty(), "no worktrees to prove empty");
1056    for candidate in &state.candidates {
1057        ensure!(
1058            (paths.contains(&candidate.worktree)
1059                || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1060                && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1061            "candidate recovery source"
1062        );
1063    }
1064    let pattern = format!("refs/heads/magi/{}/", state.short());
1065    let branches = git::git(
1066        &state.repo,
1067        &["for-each-ref", "--format=%(objectname)", &pattern],
1068    )
1069    .await?;
1070    ensure!(
1071        branches.lines().all(|head| head == state.base_commit),
1072        "unrecorded candidate branch changes"
1073    );
1074    Ok(EmptyProof {
1075        paths,
1076        _outputs: guards,
1077        _cargo: cargo_guards,
1078    })
1079}
1080
1081/// The only task-write critical section in retirement. All Git proof and
1082/// deletion happens outside it, under the run's owner/driver/output leases.
1083/// Atomic task writes make the earlier snapshot consistent; this final compare
1084/// rejects every intervening owner selection, hold, edit or history update.
1085fn commit_retirement(
1086    home: &Path,
1087    queue: &crate::queue::Queue,
1088    expected: &crate::queue::Task,
1089    state: &mut RunState,
1090    authority: &str,
1091    sources: &[String],
1092    now: Timestamp,
1093) -> Result<()> {
1094    let _write = queue.lock_task(&expected.id)?;
1095    let current = queue.get(&expected.id)?;
1096    ensure!(
1097        serde_json::to_value(&current)? == serde_json::to_value(expected)?,
1098        "task changed during retirement proof; retaining worktrees for the current owner decision"
1099    );
1100    ensure!(
1101        !current
1102            .resume_override
1103            .as_ref()
1104            .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1105            && !current
1106                .replacement
1107                .as_ref()
1108                .is_some_and(|pin| pin.run == state.id),
1109        "current owner selected this recovery source"
1110    );
1111    ensure!(
1112        !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1113        "explicitly retained recovery source"
1114    );
1115    ensure!(
1116        !crate::ask::Questions::at(home.join("questions"))
1117            .list()
1118            .iter()
1119            .any(|q| q.status.open()
1120                && (q.run == state.id
1121                    || q.run == current.id
1122                    || current.blocked_by.contains(&q.id))),
1123        "owner decision arrived during retirement proof"
1124    );
1125    let dir = home.join("recovery-lineages");
1126    std::fs::create_dir_all(&dir)?;
1127    let tmp = dir.join(format!("{}.tmp", current.id));
1128    let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1129        "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1130    std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1131    std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1132    state.retired_to = Some(authority.to_owned());
1133    state.event(
1134        "retention",
1135        format!(
1136            "empty attempt retired to {authority}; recovery sources: {}",
1137            sources.join(", ")
1138        ),
1139    );
1140    state.save_under(home)?;
1141    // No Git command or worktree removal may be added before this guard drops.
1142    Ok(())
1143}
1144
1145/// Wait for the emergency reserve to be exhausted. The normal floor requests
1146/// a checkpoint first; half the floor stops owned writers before node completion.
1147/// A failed measurement also closes admission rather than allowing blind writes.
1148pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1149    if min_free == 0 {
1150        return std::future::pending().await;
1151    }
1152    loop {
1153        for path in &paths {
1154            let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1155            match crate::disk::free_bytes(existing) {
1156                Ok(free) if free >= min_free / 2 => {}
1157                Ok(free) => {
1158                    return format!(
1159                        "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1160                    );
1161                }
1162                Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1163            }
1164        }
1165        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1166    }
1167}
1168
1169#[cfg(all(test, unix))]
1170mod tests {
1171    use super::*;
1172    use crate::{
1173        config::{Config, Disk},
1174        queue::{Source, Task},
1175        run::{Candidate, RunStatus},
1176    };
1177
1178    struct Fx {
1179        _temp: tempfile::TempDir,
1180        home: PathBuf,
1181        repo: PathBuf,
1182        bay: PathBuf,
1183        base: String,
1184    }
1185    impl Fx {
1186        async fn new() -> Self {
1187            let temp = tempfile::tempdir().unwrap();
1188            let root = temp.path().canonicalize().unwrap();
1189            let home = root.join("home");
1190            let repo = root.join("repo");
1191            let bay = root.join("worktrees");
1192            std::fs::create_dir_all(&repo).unwrap();
1193            std::fs::create_dir_all(home.join("runs")).unwrap();
1194            git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1195            git::git(&repo, &["config", "user.name", "Fixture"])
1196                .await
1197                .unwrap();
1198            git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1199                .await
1200                .unwrap();
1201            git::git(&repo, &["config", "commit.gpgsign", "false"])
1202                .await
1203                .unwrap();
1204            std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1205            std::fs::write(repo.join("source"), "base\n").unwrap();
1206            git::git(&repo, &["add", "."]).await.unwrap();
1207            git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1208            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1209            Self {
1210                _temp: temp,
1211                home,
1212                repo,
1213                bay,
1214                base,
1215            }
1216        }
1217        async fn run(&self, tag: &str, useful: bool) -> RunState {
1218            let mut cfg = Config::default();
1219            cfg.graph.worktree_root = Some(self.bay.clone());
1220            let mut state = RunState::new(
1221                self.repo.clone(),
1222                "main".into(),
1223                self.base.clone(),
1224                "task".into(),
1225                cfg,
1226            );
1227            state.id = format!("20260101-000000-{tag}");
1228            state.status = RunStatus::Blocked;
1229            state.driver_exited = true;
1230            let wt = state.worktree_root().join("cand-A");
1231            let branch = format!("magi/{tag}/A");
1232            git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1233                .await
1234                .unwrap();
1235            if useful {
1236                std::fs::write(wt.join("source"), tag).unwrap();
1237                git::git(&wt, &["add", "."]).await.unwrap();
1238                git::git(&wt, &["commit", "-m", "candidate change"])
1239                    .await
1240                    .unwrap();
1241            }
1242            state.candidates.push(Candidate {
1243                index: 0,
1244                label: 'A',
1245                agent: "fixture".into(),
1246                branch,
1247                worktree: wt,
1248                summary: String::new(),
1249                stat: String::new(),
1250                files: usize::from(useful),
1251                commits: usize::from(useful),
1252                empty: !useful,
1253                failed: None,
1254                verified_noop: None,
1255                duration_ms: 0,
1256                folded: false,
1257            });
1258            state.save_under(&self.home).unwrap();
1259            state
1260        }
1261        fn disk() -> Disk {
1262            Disk {
1263                worktree_limit_bytes: 1,
1264                worktree_reclaim_bytes: 1024 * 1024,
1265                fold_grace_secs: 0,
1266                min_free_bytes: 0,
1267                ..Disk::default()
1268            }
1269        }
1270        fn target(wt: &Path) {
1271            let target = wt.join("target");
1272            std::fs::create_dir_all(target.join("debug")).unwrap();
1273            let mut artifact = [1; 200];
1274            artifact[..4].copy_from_slice(b"\x7fELF");
1275            std::fs::write(target.join("debug/output"), artifact).unwrap();
1276            let output = target
1277                .join("debug/output")
1278                .display()
1279                .to_string()
1280                .replace('\\', "\\\\")
1281                .replace(' ', "\\ ");
1282            std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1283            std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1284            std::fs::write(
1285                target.join("CACHEDIR.TAG"),
1286                "Signature: 8a477f597d28d172789f06886806bc55d\n",
1287            )
1288            .unwrap();
1289            std::fs::write(
1290                target.join(".rustc_info.json"),
1291                "{\"rustc_fingerprint\":123}",
1292            )
1293            .unwrap();
1294        }
1295        fn now() -> Timestamp {
1296            Timestamp::now() + jiff::SignedDuration::from_secs(2)
1297        }
1298        fn task(&self, runs: &[RunState]) -> Task {
1299            let mut task = Task::new(
1300                "fixture".into(),
1301                "task".into(),
1302                self.repo.clone(),
1303                Source::Human,
1304            );
1305            task.runs = runs.iter().map(|s| s.id.clone()).collect();
1306            crate::queue::Queue::at(self.home.join("queue"))
1307                .put(&mut task)
1308                .unwrap();
1309            task
1310        }
1311    }
1312
1313    #[tokio::test]
1314    async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
1315        for remove_root in [true, false] {
1316            let f = Fx::new().await;
1317            let stale = Fx::new().await;
1318            let mut missing = stale.run("aa00", false).await;
1319            Fx::target(&missing.candidates[0].worktree);
1320            missing.save_under(&f.home).unwrap();
1321            if remove_root {
1322                std::fs::remove_dir_all(missing.worktree_root()).unwrap();
1323                // An existing non-repository proves the absent root is checked
1324                // before querying Git, even when no registrations are cached.
1325                std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
1326            } else {
1327                std::fs::remove_dir_all(&stale.repo).unwrap();
1328            }
1329            let healthy = f.run("aa01", false).await;
1330            Fx::target(&healthy.candidates[0].worktree);
1331
1332            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1333            assert!(report.failures.is_empty(), "{report:?}");
1334            assert_eq!(report.freed_bytes, 200, "{report:?}");
1335            assert!(
1336                !healthy.candidates[0]
1337                    .worktree
1338                    .join("target/debug/output")
1339                    .exists()
1340            );
1341            if !remove_root {
1342                // Without the repository, ownership cannot be validated.
1343                assert!(
1344                    missing.candidates[0]
1345                        .worktree
1346                        .join("target/debug/output")
1347                        .exists()
1348                );
1349            }
1350        }
1351    }
1352
1353    #[tokio::test]
1354    async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
1355        let f = Fx::new().await;
1356        let mut missing = f.run("aa00", false).await;
1357        Fx::target(&missing.candidates[0].worktree);
1358        missing.repo = missing.candidates[0].worktree.clone();
1359        missing.save_under(&f.home).unwrap();
1360        std::fs::remove_dir_all(&missing.repo).unwrap();
1361        assert!(missing.worktree_root().is_dir());
1362        assert!(f.repo.is_dir());
1363
1364        let healthy = f.run("aa01", false).await;
1365        Fx::target(&healthy.candidates[0].worktree);
1366
1367        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1368        assert!(report.failures.is_empty(), "{report:?}");
1369        assert_eq!(report.freed_bytes, 200, "{report:?}");
1370        assert!(
1371            !healthy.candidates[0]
1372                .worktree
1373                .join("target/debug/output")
1374                .exists()
1375        );
1376    }
1377
1378    #[tokio::test]
1379    async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
1380        let f = Fx::new().await;
1381        let broken = Fx::new().await;
1382        let mut invalid = broken.run("aa00", false).await;
1383        Fx::target(&invalid.candidates[0].worktree);
1384        invalid.save_under(&f.home).unwrap();
1385        std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
1386        let healthy = f.run("aa01", false).await;
1387        Fx::target(&healthy.candidates[0].worktree);
1388
1389        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1390        assert_eq!(report.failures.len(), 1, "{report:?}");
1391        assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
1392        assert_eq!(report.freed_bytes, 200, "{report:?}");
1393        assert!(
1394            !healthy.candidates[0]
1395                .worktree
1396                .join("target/debug/output")
1397                .exists()
1398        );
1399        assert!(
1400            invalid.candidates[0]
1401                .worktree
1402                .join("target/debug/output")
1403                .exists()
1404        );
1405    }
1406
1407    async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
1408        let path = reconcile::run(&f.home, &task.id, &source.id, None)
1409            .await
1410            .unwrap();
1411        serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
1412    }
1413
1414    #[tokio::test]
1415    async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
1416        use crate::{
1417            chat_control::{Operation, dispatch},
1418            talk::{self, Talks},
1419        };
1420        let f = Fx::new().await;
1421        let source = f.run("ae00", true).await;
1422        let clean = f.run("ae01", false).await;
1423        let queue = crate::queue::Queue::at(f.home.join("queue"));
1424        let talks = Talks::at(f.home.join("talks"));
1425        let open_talk = |id: &str| -> talk::Talk {
1426            let mut conversation = serde_json::from_value(serde_json::json!({
1427                "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
1428                "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
1429                "seat":crate::agent::SeatState::new("talk","worker",1)
1430            }))
1431            .unwrap();
1432            talks.put(&mut conversation).unwrap();
1433            conversation
1434        };
1435        let conversation = open_talk("talk-owner");
1436        let other = open_talk("talk-other");
1437        // A task this chat did not file: the owner's request covers it too.
1438        let task = f.task(&[source.clone(), clean.clone()]);
1439        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1440        let operation = |apply: Option<&str>| Operation::ReconcileRetention {
1441            task: task.id.clone(),
1442            authority: source.id.clone(),
1443            apply: apply.map(str::to_owned),
1444        };
1445        // A plan saved outside the chat is not applied through it.
1446        let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
1447            .await
1448            .unwrap();
1449        let legacy: serde_json::Value =
1450            serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
1451        let legacy_id = legacy["id"].as_str().unwrap();
1452        assert!(
1453            dispatch(
1454                &queue,
1455                &f.home,
1456                &talks,
1457                &conversation.id,
1458                operation(Some(legacy_id))
1459            )
1460            .await
1461            .is_err()
1462        );
1463        let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
1464            .await
1465            .unwrap();
1466        assert!(summary.contains("Complete review summary"));
1467        assert!(summary.contains(&source.id) && summary.contains(&clean.id));
1468        let root = f.home.join("retention-reconciliations");
1469        let displayed: serde_json::Value =
1470            serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
1471        let id = displayed["plan"].as_str().unwrap();
1472        let dir = root.join(id);
1473        let plan: serde_json::Value =
1474            serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
1475        assert_eq!(plan["version"], 2);
1476        assert!(clean.candidates[0].worktree.exists());
1477        assert!(!dir.join("approved.json").exists());
1478        // The plan stays bound to the conversation it was saved for.
1479        assert!(
1480            dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
1481                .await
1482                .is_err()
1483        );
1484        assert!(!dir.join("approved.json").exists());
1485        // No owner phrase or quote: the request itself applies the plan. An
1486        // interruption after the durable fence keeps the audit for a retry.
1487        let error = reconcile::interrupt_chat_after_fence(
1488            &f.home,
1489            &task,
1490            id,
1491            reconcile::ChatRequest {
1492                talks: &talks,
1493                conversation: &conversation.id,
1494            },
1495        )
1496        .await
1497        .unwrap_err();
1498        assert!(
1499            error.to_string().contains("simulated interruption"),
1500            "{error:#}"
1501        );
1502        assert!(clean.candidates[0].worktree.exists());
1503        assert!(dir.join("approved.json").exists());
1504        dispatch(
1505            &queue,
1506            &f.home,
1507            &talks,
1508            &conversation.id,
1509            operation(Some(id)),
1510        )
1511        .await
1512        .unwrap();
1513        assert!(!clean.candidates[0].worktree.exists());
1514        assert!(source.candidates[0].worktree.exists());
1515        let audit: serde_json::Value =
1516            serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
1517        assert_eq!(audit["version"], 2);
1518        assert_eq!(
1519            audit["owner"],
1520            serde_json::json!({"conversation": conversation.id})
1521        );
1522        dispatch(
1523            &queue,
1524            &f.home,
1525            &talks,
1526            &conversation.id,
1527            operation(Some(id)),
1528        )
1529        .await
1530        .unwrap();
1531    }
1532
1533    #[tokio::test]
1534    async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
1535        let f = Fx::new().await;
1536        let source = f.run("af00", true).await;
1537        let mut clean = f.run("af01", false).await;
1538        let mut dirty = f.run("af02", false).await;
1539        for state in [&mut clean, &mut dirty] {
1540            state.status = RunStatus::Prep;
1541            state.prep_complete = Some(false);
1542            state.candidates[0].empty = false;
1543            state.save_under(&f.home).unwrap();
1544        }
1545        std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
1546        let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
1547        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1548        let plan = reconciliation_plan(&f, &task, &source).await;
1549        let selected: Vec<_> = plan["candidates"]
1550            .as_array()
1551            .unwrap()
1552            .iter()
1553            .filter(|e| e["selected"] == true)
1554            .collect();
1555        assert_eq!(selected.len(), 1, "{plan:#}");
1556        assert_eq!(selected[0]["run"], clean.id);
1557        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1558            .await
1559            .unwrap();
1560        assert!(!clean.candidates[0].worktree.exists());
1561        assert!(dirty.candidates[0].worktree.exists());
1562        assert!(
1563            RunState::load_under(&dirty.id, &f.home)
1564                .unwrap()
1565                .retired_to
1566                .is_none()
1567        );
1568    }
1569
1570    #[tokio::test]
1571    async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
1572        let f = Fx::new().await;
1573        let mut source = f.run("aa00", true).await;
1574        let alternative = f.run("aa01", true).await;
1575        let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
1576            .await
1577            .unwrap();
1578        source.reviewed_commits = Some(vec![head.clone()]);
1579        source.save_under(&f.home).unwrap();
1580        let mut runs = vec![source.clone(), alternative.clone()];
1581        for i in 2..14 {
1582            runs.push(f.run(&format!("aa{i:02}"), false).await);
1583        }
1584        let mut task = f.task(&runs);
1585        task.pr_recovery = Some(crate::pr_recovery::Selection {
1586            url: "https://example.invalid/pull/1".into(),
1587            branch: "selected".into(),
1588            head: head.clone(),
1589            source_head: Some(head.clone()),
1590            source_run: Some(source.id.clone()),
1591            source_proof: None,
1592            disposition: Some(crate::pr_recovery::Disposition::Land),
1593            instruction: task.instruction.clone(),
1594            approved_at: Timestamp::now(),
1595        });
1596        source.pr_recovery = task.pr_recovery.clone();
1597        source.save_under(&f.home).unwrap();
1598        runs[0] = source.clone();
1599        crate::queue::Queue::at(f.home.join("queue"))
1600            .put(&mut task)
1601            .unwrap();
1602        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1603        let task_before = serde_json::to_value(&task).unwrap();
1604        let mut report = Report::default();
1605        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
1606        assert_eq!(
1607            report.retired, 0,
1608            "automatic cleanup must still refuse blanket markers"
1609        );
1610        assert!(
1611            report
1612                .retained
1613                .iter()
1614                .any(|reason| reason.contains("explicitly retained"))
1615        );
1616        let mut total = 0;
1617        for expected in [5, 5, 2, 0] {
1618            let plan = reconciliation_plan(&f, &task, &source).await;
1619            let selected: Vec<_> = plan["candidates"]
1620                .as_array()
1621                .unwrap()
1622                .iter()
1623                .filter(|e| e["selected"] == true)
1624                .collect();
1625            assert_eq!(selected.len(), expected, "{plan:#}");
1626            for e in &selected {
1627                assert!(
1628                    RunState::load_under(e["run"].as_str().unwrap(), &f.home)
1629                        .unwrap()
1630                        .retired_to
1631                        .is_none()
1632                );
1633            }
1634            let operation = plan["id"].as_str().unwrap();
1635            for _ in 0..2 {
1636                reconcile::run(&f.home, &task.id, &source.id, Some(operation))
1637                    .await
1638                    .unwrap();
1639            }
1640            total += expected;
1641        }
1642        assert_eq!(total, 12);
1643        for run in &runs {
1644            let dir = f.home.join("runs").join(&run.id);
1645            assert!(dir.join("run.json").exists());
1646            assert!(dir.join("retain-recovery-sources").exists());
1647            assert!(
1648                git::branch_exists(&f.repo, &run.candidates[0].branch)
1649                    .await
1650                    .unwrap()
1651            );
1652        }
1653        assert!(source.candidates[0].worktree.exists());
1654        assert!(alternative.candidates[0].worktree.exists());
1655        assert_eq!(
1656            git::rev_parse(&f.repo, &source.candidates[0].branch)
1657                .await
1658                .unwrap(),
1659            head
1660        );
1661        assert_eq!(
1662            serde_json::to_value(
1663                crate::queue::Queue::at(f.home.join("queue"))
1664                    .get(&task.id)
1665                    .unwrap()
1666            )
1667            .unwrap(),
1668            task_before
1669        );
1670        assert_eq!(
1671            runs.iter()
1672                .filter(|r| RunState::load_under(&r.id, &f.home)
1673                    .unwrap()
1674                    .retired_to
1675                    .is_none())
1676                .count(),
1677            2
1678        );
1679    }
1680
1681    #[tokio::test]
1682    async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
1683        let f = Fx::new().await;
1684        let source = f.run("ab00", true).await;
1685        let mut runs = vec![source.clone()];
1686        for i in 1..9 {
1687            runs.push(f.run(&format!("ab{i:02}"), false).await);
1688        }
1689        std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
1690        std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
1691        std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
1692        std::fs::write(
1693            runs[3].candidates[0].worktree.join("scratch/source"),
1694            "ignored source",
1695        )
1696        .unwrap();
1697        std::fs::write(
1698            runs[4].worktree_root().join("unknown"),
1699            "unregistered source",
1700        )
1701        .unwrap();
1702        let checkpoint = git::git(
1703            &runs[5].candidates[0].worktree,
1704            &[
1705                "rev-parse",
1706                "--path-format=absolute",
1707                "--git-path",
1708                "sequencer",
1709            ],
1710        )
1711        .await
1712        .unwrap();
1713        std::fs::create_dir(checkpoint.trim()).unwrap();
1714        runs[6].driver_exited = false;
1715        runs[6].driver_pid = Some(std::process::id());
1716        runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
1717        runs[6].save_under(&f.home).unwrap();
1718        runs[7].driver_exited = false;
1719        runs[7].driver_pid = None;
1720        runs[7].driver_claimed = None; // Legacy record without first-claim evidence.
1721        runs[7].save_under(&f.home).unwrap();
1722        assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
1723
1724        let hidden = &runs[8].candidates[0].worktree;
1725        git::git(hidden, &["update-index", "--assume-unchanged", "source"])
1726            .await
1727            .unwrap();
1728        std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
1729        assert!(
1730            git::git(hidden, &["status", "--porcelain"])
1731                .await
1732                .unwrap()
1733                .is_empty()
1734        );
1735        let task = f.task(&runs);
1736        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1737        let plan = reconciliation_plan(&f, &task, &source).await;
1738        assert!(
1739            plan["candidates"]
1740                .as_array()
1741                .unwrap()
1742                .iter()
1743                .all(|e| e["selected"] == false),
1744            "{plan:#}"
1745        );
1746        for run in &runs {
1747            assert!(run.candidates[0].worktree.exists());
1748        }
1749    }
1750
1751    #[tokio::test]
1752    async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
1753        let f = Fx::new().await;
1754        let source = f.run("ac00", true).await;
1755        let empty = f.run("ac01", false).await;
1756        let task = f.task(&[source.clone(), empty.clone()]);
1757        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1758        let plan = reconciliation_plan(&f, &task, &source).await;
1759        std::fs::write(
1760            f.home.join("runs").join(&source.id).join("new-report"),
1761            "new review checkpoint",
1762        )
1763        .unwrap();
1764        assert!(
1765            reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1766                .await
1767                .is_err()
1768        );
1769        assert!(empty.candidates[0].worktree.exists());
1770        assert!(
1771            RunState::load_under(&empty.id, &f.home)
1772                .unwrap()
1773                .retired_to
1774                .is_none()
1775        );
1776        let marker = f
1777            .home
1778            .join("runs")
1779            .join(&empty.id)
1780            .join("retain-recovery-sources");
1781        std::fs::write(&marker, "different owner").unwrap();
1782        let plan = reconciliation_plan(&f, &task, &source).await;
1783        assert!(
1784            plan["candidates"]
1785                .as_array()
1786                .unwrap()
1787                .iter()
1788                .all(|e| e["selected"] == false)
1789        );
1790    }
1791
1792    #[tokio::test]
1793    async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
1794        for partial in [false, true] {
1795            let f = Fx::new().await;
1796            let source = f.run("ab10", true).await;
1797            let first = f.run("ab11", false).await;
1798            let second = f.run("ab12", false).await;
1799            let task = f.task(&[source.clone(), first.clone(), second.clone()]);
1800            crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1801            let plan = reconciliation_plan(&f, &task, &source).await;
1802            let id = plan["id"].as_str().unwrap();
1803            let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
1804            // Matching bytes without this operation's fence are not trusted.
1805            std::fs::write(&staging, b"").unwrap();
1806            assert!(
1807                reconcile::run(&f.home, &task.id, &source.id, Some(id))
1808                    .await
1809                    .is_err()
1810            );
1811            assert!(staging.exists());
1812            std::fs::remove_file(&staging).unwrap();
1813            let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
1814                .await
1815                .unwrap_err();
1816            assert!(
1817                error.to_string().contains("before run state rename"),
1818                "{error:#}"
1819            );
1820            assert!(
1821                RunState::load_under(&first.id, &f.home)
1822                    .unwrap()
1823                    .retired_to
1824                    .is_some()
1825            );
1826            assert!(
1827                RunState::load_under(&second.id, &f.home)
1828                    .unwrap()
1829                    .retired_to
1830                    .is_none()
1831            );
1832            assert!(first.candidates[0].worktree.exists());
1833            assert!(second.candidates[0].worktree.exists());
1834            let interrupted = std::fs::read(&staging).unwrap();
1835            // An unrelated file must survive a refused retry unchanged.
1836            std::fs::write(&staging, b"unrelated recovery source").unwrap();
1837            assert!(
1838                reconcile::run(&f.home, &task.id, &source.id, Some(id))
1839                    .await
1840                    .is_err()
1841            );
1842            assert_eq!(
1843                std::fs::read(&staging).unwrap(),
1844                b"unrelated recovery source"
1845            );
1846            std::fs::write(&staging, interrupted).unwrap();
1847            reconcile::run(&f.home, &task.id, &source.id, Some(id))
1848                .await
1849                .unwrap();
1850            assert!(!staging.exists());
1851            assert!(!first.candidates[0].worktree.exists());
1852            assert!(!second.candidates[0].worktree.exists());
1853            assert_eq!(
1854                RunState::load_under(&second.id, &f.home)
1855                    .unwrap()
1856                    .retired_to,
1857                Some(source.id)
1858            );
1859        }
1860    }
1861
1862    #[tokio::test]
1863    async fn reconciliation_resumes_only_its_exact_durable_fence() {
1864        let f = Fx::new().await;
1865        let source = f.run("ad00", true).await;
1866        let empty = f.run("ad01", false).await;
1867        let task = f.task(&[source.clone(), empty.clone()]);
1868        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1869        let plan = reconciliation_plan(&f, &task, &source).await;
1870        let id = plan["id"].as_str().unwrap();
1871        assert!(
1872            reconcile::interrupt_after_fence(&f.home, &task, id)
1873                .await
1874                .is_err()
1875        );
1876        assert!(empty.candidates[0].worktree.exists());
1877        assert_eq!(
1878            RunState::load_under(&empty.id, &f.home)
1879                .unwrap()
1880                .retired_to
1881                .as_ref(),
1882            Some(&source.id)
1883        );
1884        reconcile::run(&f.home, &task.id, &source.id, Some(id))
1885            .await
1886            .unwrap();
1887        assert!(!empty.candidates[0].worktree.exists());
1888        // Model an interrupted removal: restore precisely the approved base worktree.
1889        git::worktree_add_branch(
1890            &f.repo,
1891            &empty.candidates[0].worktree,
1892            "restore-empty",
1893            &f.base,
1894        )
1895        .await
1896        .unwrap();
1897        // Recreated Git administration differs from approval and is refused.
1898        assert!(
1899            reconcile::run(&f.home, &task.id, &source.id, Some(id))
1900                .await
1901                .is_err()
1902        );
1903        assert!(empty.candidates[0].worktree.exists());
1904        let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
1905        changed.prep_failure = Some("new owner edit".into());
1906        changed.save_under(&f.home).unwrap();
1907        assert!(
1908            reconcile::run(&f.home, &task.id, &source.id, Some(id))
1909                .await
1910                .is_err()
1911        );
1912    }
1913
1914    #[tokio::test]
1915    async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
1916        let f = Fx::new().await;
1917        let source = f.run("ae00", true).await;
1918        let mut prep = f.run("ae01", false).await;
1919        let candidate = prep.candidates.pop().unwrap();
1920        prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
1921        prep.save_under(&f.home).unwrap();
1922        let reset = f.run("ae02", false).await;
1923        let wt = &reset.candidates[0].worktree;
1924        std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
1925        git::git(wt, &["add", "."]).await.unwrap();
1926        git::git(wt, &["commit", "-m", "unique implementation"])
1927            .await
1928            .unwrap();
1929        git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
1930        let unknown = f.run("ae03", false).await;
1931        let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
1932        let mut raw: serde_json::Value =
1933            serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
1934        raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
1935        std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
1936        let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
1937        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1938        let plan = reconciliation_plan(&f, &task, &source).await;
1939        let selected: Vec<_> = plan["candidates"]
1940            .as_array()
1941            .unwrap()
1942            .iter()
1943            .filter(|e| e["selected"] == true)
1944            .collect();
1945        assert_eq!(selected.len(), 1, "{plan:#}");
1946        assert_eq!(selected[0]["run"], prep.id);
1947        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1948            .await
1949            .unwrap();
1950        assert!(!candidate.worktree.exists());
1951        assert!(reset.candidates[0].worktree.exists());
1952        assert!(unknown.candidates[0].worktree.exists());
1953    }
1954
1955    #[tokio::test]
1956    async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
1957        let f = Fx::new().await;
1958        let mut state = f.run("aaaa", true).await;
1959        let candidate = state.candidates[0].worktree.clone();
1960        std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
1961        Fx::target(&candidate);
1962        for seat in ["review-1", "review-2"] {
1963            let wt = state.worktree_root().join(seat);
1964            git::worktree_add_detached(&f.repo, &wt, &f.base)
1965                .await
1966                .unwrap();
1967            Fx::target(&wt);
1968        }
1969        state.status = RunStatus::Landing;
1970        state.save_under(&f.home).unwrap();
1971        let second = Fx::new().await;
1972        let other = second.run("bbbb", true).await;
1973        // A global store may contain repositories with different worktree roots.
1974        let mut imported = other.clone();
1975        imported.save_under(&f.home).unwrap();
1976        Fx::target(&other.candidates[0].worktree);
1977        let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
1978        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1979        assert!(
1980            candidate.join("target/debug/output").exists(),
1981            "active writer protected"
1982        );
1983        assert!(
1984            !state
1985                .worktree_root()
1986                .join("review-1/target/debug/output")
1987                .exists()
1988        );
1989        assert!(
1990            !state
1991                .worktree_root()
1992                .join("review-2/target/debug/output")
1993                .exists()
1994        );
1995        assert!(
1996            !other.candidates[0]
1997                .worktree
1998                .join("target/debug/output")
1999                .exists()
2000        );
2001        assert!(out.freed_bytes >= 600);
2002        drop(guard);
2003        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2004        assert!(out.freed_bytes >= 200);
2005        assert_eq!(
2006            std::fs::read_to_string(candidate.join("source")).unwrap(),
2007            "uncommitted work"
2008        );
2009        assert!(
2010            git::branch_exists(&f.repo, &state.candidates[0].branch)
2011                .await
2012                .unwrap()
2013        );
2014    }
2015
2016    #[tokio::test]
2017    async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
2018        let f = Fx::new().await;
2019        let state = f.run("cc01", false).await;
2020        let wt = &state.candidates[0].worktree;
2021        Fx::target(wt);
2022        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2023            let dir = wt.join("target").join(dir);
2024            std::fs::create_dir_all(&dir).unwrap();
2025            std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
2026        }
2027        // A dep-info dependency is an input, never deletion authority.
2028        let manifest = wt.join("target/debug/output.d");
2029        let output = wt.join("target/debug/output");
2030        std::fs::write(
2031            &manifest,
2032            format!(
2033                "{}: {}\n",
2034                output.display(),
2035                wt.join("target/debug/notes.txt").display()
2036            ),
2037        )
2038        .unwrap();
2039        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2040        assert_eq!(report.freed_bytes, 200);
2041        assert!(!output.exists());
2042        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2043            assert_eq!(
2044                std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
2045                "user evidence"
2046            );
2047        }
2048        // Metadata remains available for inspection and future safe rebuilds.
2049        assert!(manifest.exists());
2050        assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
2051        std::fs::write(&output, "user note replacing an old binary").unwrap();
2052        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2053        assert_eq!(report.freed_bytes, 0);
2054        assert_eq!(
2055            std::fs::read_to_string(&output).unwrap(),
2056            "user note replacing an old binary"
2057        );
2058    }
2059
2060    #[tokio::test]
2061    async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
2062        let f = Fx::new().await;
2063        let state = f.run("cc02", false).await;
2064        let wt = &state.candidates[0].worktree;
2065        Fx::target(wt);
2066        let mut disk = Fx::disk();
2067        disk.worktree_limit_bytes = u64::MAX;
2068        let now = Fx::now();
2069        let first = sweep(&f.home, &disk, now).await;
2070        std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
2071        assert_eq!(
2072            sweep(&f.home, &disk, now).await.measured_bytes,
2073            first.measured_bytes
2074        );
2075        assert!(
2076            sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
2077                .await
2078                .measured_bytes
2079                > first.measured_bytes
2080        );
2081        disk.worktree_limit_bytes = 0;
2082        assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
2083    }
2084
2085    #[tokio::test]
2086    async fn unfinished_source_fold_reports_its_retention_reason() {
2087        let f = Fx::new().await;
2088        let mut source = f.run("cc03", true).await;
2089        let task = f.task(&[source.clone()]);
2090        let error = crate::graph::fold_run(&mut source, true, &f.home)
2091            .await
2092            .unwrap_err();
2093        assert!(error.to_string().contains(&task.id));
2094        assert!(source.candidates[0].worktree.exists());
2095        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2096        let error = crate::graph::fold_run(&mut source, true, &f.home)
2097            .await
2098            .unwrap_err();
2099        assert!(error.to_string().contains("explicitly retained"));
2100    }
2101
2102    #[tokio::test]
2103    async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
2104        let f = Fx::new().await;
2105        let mut source = f.run("cc04", true).await;
2106        let mut task = f.task(&[source.clone()]);
2107        task.hold_manual(Some("queue settlement pending".into()));
2108        let queue = crate::queue::Queue::at(f.home.join("queue"));
2109        queue.put(&mut task).unwrap();
2110        source.merge = Some(crate::run::MergeOutcome {
2111            mode: crate::config::MergeMode::Pr,
2112            ok: true,
2113            detail: "confirmed merged".into(),
2114            empty: false,
2115        });
2116        // Publication success and terminal labels alone are insufficient.
2117        for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
2118            source.status = status;
2119            assert!(
2120                crate::graph::fold_run(&mut source, true, &f.home)
2121                    .await
2122                    .is_err()
2123            );
2124            assert!(source.candidates[0].worktree.exists());
2125        }
2126        git::git(
2127            &f.repo,
2128            &["merge", "--ff-only", &source.candidates[0].branch],
2129        )
2130        .await
2131        .unwrap();
2132        git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2133            .await
2134            .unwrap();
2135        source.status = RunStatus::Merged;
2136        source.save_under(&f.home).unwrap();
2137        assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
2138        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2139        assert!(
2140            crate::graph::fold_run(&mut source, true, &f.home)
2141                .await
2142                .unwrap_err()
2143                .to_string()
2144                .contains("explicitly retained")
2145        );
2146        // The fixture's explicit marker is removed only to exercise ordinary
2147        // confirmed-merge cleanup; no production cleanup removes such markers.
2148        std::fs::remove_file(
2149            f.home
2150                .join("runs")
2151                .join(&source.id)
2152                .join("retain-recovery-sources"),
2153        )
2154        .unwrap();
2155        let (folded, unreadable) = crate::clean::fold_due(
2156            &f.home.join("runs"),
2157            &f.home,
2158            &f.bay,
2159            &Fx::disk(),
2160            Fx::now(),
2161        )
2162        .await
2163        .unwrap();
2164        assert_eq!((folded, unreadable), (1, 0));
2165        assert!(!source.candidates[0].worktree.exists());
2166        let evidence = f
2167            .home
2168            .join("runs")
2169            .join(&source.id)
2170            .join("artifacts/fold-evidence");
2171        let names: Vec<_> = std::fs::read_dir(evidence)
2172            .unwrap()
2173            .map(|entry| entry.unwrap().file_name())
2174            .collect();
2175        assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
2176        assert_eq!(
2177            queue.get(&task.id).unwrap().status,
2178            crate::queue::TaskStatus::Held
2179        );
2180    }
2181
2182    #[tokio::test]
2183    async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
2184        for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
2185            let f = Fx::new().await;
2186            let mut state = f.run("cc06", true).await;
2187            let candidate = state.candidates[0].clone();
2188            let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2189            std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
2190            git::rescue_commit(&candidate.worktree, "second candidate change")
2191                .await
2192                .unwrap();
2193            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2194            state.tally = Some(crate::run::Tally {
2195                first_choice: Default::default(),
2196                borda: Default::default(),
2197                winner: 'A',
2198                rankings: 0,
2199                unanimous_initial: true,
2200                deliberated: false,
2201                changed_votes: 0,
2202                unanimous_final: true,
2203                tie_break: None,
2204                judges: 0,
2205                present: 0,
2206                quorum: 0,
2207                met_quorum: true,
2208                uncontested: Some("fixture".into()),
2209            });
2210            // A reviewer pinned to an intermediate candidate commit must also
2211            // be reclaimable, although neither commit ID lands in the squash.
2212            let reviewer = state.worktree_root().join("review-1");
2213            git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
2214                .await
2215                .unwrap();
2216            // Concurrent base changes and later base advances prevent a mere
2217            // equality check against the current base tree from passing.
2218            std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
2219            git::rescue_commit(&f.repo, "advance before squash")
2220                .await
2221                .unwrap();
2222            let published = if variant == "partial" {
2223                &intermediate
2224            } else {
2225                &head
2226            };
2227            git::git(&f.repo, &["merge", "--squash", published])
2228                .await
2229                .unwrap();
2230            git::git(&f.repo, &["commit", "-m", "squash candidate"])
2231                .await
2232                .unwrap();
2233            let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
2234            assert!(
2235                !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
2236                    .await
2237                    .unwrap()
2238                    .ok()
2239            );
2240            std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
2241            git::rescue_commit(&f.repo, "advance after squash")
2242                .await
2243                .unwrap();
2244            git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2245                .await
2246                .unwrap();
2247            if matches!(variant, "checkpoint" | "reviewer") {
2248                // A reset hides unique work from HEAD, but not its reflog.
2249                std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
2250                git::rescue_commit(&reviewer, "unpublished checkpoint")
2251                    .await
2252                    .unwrap();
2253                if variant == "checkpoint" {
2254                    git::git(&reviewer, &["reset", "--hard", &intermediate])
2255                        .await
2256                        .unwrap();
2257                }
2258            }
2259            if variant == "loser" {
2260                let mut loser = candidate.clone();
2261                loser.label = 'B';
2262                loser.branch = "magi/cc06/B".into();
2263                loser.worktree = state.worktree_root().join("cand-B");
2264                git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
2265                    .await
2266                    .unwrap();
2267                std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
2268                git::rescue_commit(&loser.worktree, "independent loser")
2269                    .await
2270                    .unwrap();
2271                state.candidates.push(loser);
2272            }
2273            state.status = RunStatus::Merged;
2274            state.merge = Some(crate::run::MergeOutcome {
2275                mode: crate::config::MergeMode::Pr,
2276                ok: true,
2277                detail: "candidate A squash merged".into(),
2278                empty: false,
2279            });
2280            state.save_under(&f.home).unwrap();
2281            let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
2282            let result = crate::graph::fold_run(&mut state, true, &f.home).await;
2283            result.unwrap();
2284            assert!(
2285                !candidate.worktree.exists() && !reviewer.exists(),
2286                "{variant}"
2287            );
2288            assert!(
2289                !git::branch_exists(&f.repo, &candidate.branch)
2290                    .await
2291                    .unwrap()
2292            );
2293            let artifacts = f
2294                .home
2295                .join("runs")
2296                .join(&state.id)
2297                .join("artifacts/fold-evidence");
2298            assert_eq!(
2299                artifacts.join(format!("{head}.patch")).exists(),
2300                variant == "partial",
2301                "only a partially published winner needs evidence: {variant}"
2302            );
2303            assert_eq!(
2304                artifacts.join(format!("{intermediate}.patch")).exists(),
2305                variant == "partial",
2306                "squash proof also covers intermediate reviewer tips: {variant}"
2307            );
2308            let patches: Vec<_> = std::fs::read_dir(&artifacts)
2309                .unwrap()
2310                .map(|entry| entry.unwrap().path())
2311                .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
2312                .collect();
2313            let expected = match variant {
2314                "partial" => 2,
2315                "loser" | "reviewer" => 1,
2316                _ => 0,
2317            };
2318            assert_eq!(patches.len(), expected, "{variant}");
2319            if variant == "loser" {
2320                assert!(
2321                    std::fs::read_to_string(&patches[0])
2322                        .unwrap()
2323                        .contains("losing candidate")
2324                );
2325            }
2326            assert!(artifacts.join("manifest.json").exists());
2327            if variant == "reviewer" {
2328                assert!(
2329                    std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
2330                        .unwrap()
2331                        .contains("reviewer checkpoint")
2332                );
2333            }
2334            crate::graph::fold_run(&mut state, true, &f.home)
2335                .await
2336                .unwrap();
2337        }
2338    }
2339
2340    #[tokio::test]
2341    async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
2342        for missing in [true, false] {
2343            let f = Fx::new().await;
2344            let mut state = f.run("cc05", true).await;
2345            // A fetched base exists, but these local review changes were not
2346            // incorporated by ancestry, so their recovery evidence is required.
2347            git::git(
2348                &f.repo,
2349                &["update-ref", "refs/remotes/origin/main", &f.base],
2350            )
2351            .await
2352            .unwrap();
2353            let candidate = state.candidates[0].clone();
2354            if !missing {
2355                std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
2356                git::rescue_commit(&f.repo, "advance base before rebase")
2357                    .await
2358                    .unwrap();
2359                git::git(&candidate.worktree, &["rebase", "main"])
2360                    .await
2361                    .unwrap();
2362                // Match the runner's post-rebase synchronization, which
2363                // clears the merge backend's transient AUTO_MERGE checkpoint.
2364                git::sync_to_head(&candidate.worktree).await.unwrap();
2365            }
2366            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2367            // Model review-only state: no implementation-time snapshot exists.
2368            state.reviewed_commits = Some(Vec::new());
2369            state.candidates[0].worktree = state.worktree_root().join("under-review");
2370            git::git(
2371                &f.repo,
2372                &[
2373                    "worktree",
2374                    "move",
2375                    candidate.worktree.to_str().unwrap(),
2376                    state.candidates[0].worktree.to_str().unwrap(),
2377                ],
2378            )
2379            .await
2380            .unwrap();
2381            let reviewer = state.worktree_root().join("review-1");
2382            git::worktree_add_detached(&f.repo, &reviewer, &head)
2383                .await
2384                .unwrap();
2385            // A reset/rebase can leave unpublished recovery history. It must
2386            // not keep any remaining owned worktree alive indefinitely.
2387            std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
2388            git::rescue_commit(&reviewer, "old review history")
2389                .await
2390                .unwrap();
2391            git::git(&reviewer, &["reset", "--hard", &head])
2392                .await
2393                .unwrap();
2394            if missing {
2395                std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
2396            }
2397            let run_dir = f.home.join("runs").join(&state.id);
2398            let artifacts = run_dir.join("artifacts");
2399            std::fs::create_dir_all(&artifacts).unwrap();
2400            std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
2401            if missing {
2402                std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
2403            }
2404            state.status = RunStatus::Merged;
2405            state.merge = Some(crate::run::MergeOutcome {
2406                mode: crate::config::MergeMode::Pr,
2407                ok: true,
2408                detail: "confirmed squash merge".into(),
2409                empty: false,
2410            });
2411            state.save_under(&f.home).unwrap();
2412            // Unexpected external paths remain protected, even if absent.
2413            let owned = state.candidates[0].worktree.clone();
2414            state.candidates[0].worktree = f.home.join("missing-external");
2415            assert!(
2416                crate::graph::fold_run(&mut state, true, &f.home)
2417                    .await
2418                    .unwrap_err()
2419                    .to_string()
2420                    .contains("outside the inventoried")
2421            );
2422            state.candidates[0].worktree = owned;
2423            // A live output borrower blocks the entire preflight.
2424            let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
2425            assert!(
2426                crate::graph::fold_run(&mut state, true, &f.home)
2427                    .await
2428                    .is_err()
2429            );
2430            assert!(reviewer.exists());
2431            drop(guard);
2432            // Failure to persist evidence must not cause partial removal.
2433            let evidence = artifacts.join("fold-evidence");
2434            std::fs::write(&evidence, "blocked").unwrap();
2435            assert!(
2436                crate::graph::fold_run(&mut state, true, &f.home)
2437                    .await
2438                    .is_err()
2439            );
2440            assert!(reviewer.exists());
2441            std::fs::remove_file(&evidence).unwrap();
2442            std::fs::create_dir_all(&evidence).unwrap();
2443            let oversized = evidence.join("capacity-fixture");
2444            std::fs::File::create(&oversized)
2445                .unwrap()
2446                .set_len(64 * 1024 * 1024)
2447                .unwrap();
2448            assert!(
2449                crate::graph::fold_run(&mut state, true, &f.home)
2450                    .await
2451                    .is_err()
2452            );
2453            assert!(reviewer.exists());
2454            std::fs::remove_file(oversized).unwrap();
2455            // Ignored recovery files and unfinished Git operations still hold
2456            // all sources, even for a confirmed merge.
2457            std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
2458            std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
2459            assert!(
2460                crate::graph::fold_run(&mut state, true, &f.home)
2461                    .await
2462                    .is_err()
2463            );
2464            std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
2465            let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
2466                .await
2467                .unwrap();
2468            std::fs::write(&checkpoint, &head).unwrap();
2469            assert!(
2470                crate::graph::fold_run(&mut state, true, &f.home)
2471                    .await
2472                    .is_err()
2473            );
2474            std::fs::remove_file(checkpoint).unwrap();
2475            let (folded, unreadable) = crate::clean::fold_due(
2476                &f.home.join("runs"),
2477                &f.home,
2478                &f.bay,
2479                &Fx::disk(),
2480                Fx::now(),
2481            )
2482            .await
2483            .unwrap();
2484            if folded != 1 {
2485                panic!(
2486                    "normal cleanup failed: {:#}",
2487                    crate::graph::fold_run(&mut state, true, &f.home)
2488                        .await
2489                        .unwrap_err()
2490                );
2491            }
2492            assert_eq!((folded, unreadable), (1, 0));
2493            assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
2494            state = RunState::load_under(&state.id, &f.home).unwrap();
2495            assert!(state.candidates[0].folded);
2496            let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
2497            crate::graph::fold_run(&mut state, true, &f.home)
2498                .await
2499                .unwrap();
2500            assert_eq!(
2501                std::fs::read(evidence.join("manifest.json")).unwrap(),
2502                manifest
2503            );
2504            git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
2505                .await
2506                .unwrap();
2507            git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
2508            assert!(
2509                !git::git_raw(&f.repo, &["cat-file", "-e", &head])
2510                    .await
2511                    .unwrap()
2512                    .ok()
2513            );
2514            // Restore the modified file into a fresh repository from artifact
2515            // bytes alone; no original commit or blob is consulted.
2516            let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
2517            let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
2518                .as_str()
2519                .unwrap();
2520            let recovered = f.home.join("recovered");
2521            std::fs::create_dir_all(&recovered).unwrap();
2522            git::git(&recovered, &["init"]).await.unwrap();
2523            std::fs::write(
2524                recovered.join("source"),
2525                std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
2526            )
2527            .unwrap();
2528            git::git(
2529                &recovered,
2530                &[
2531                    "apply",
2532                    evidence.join(format!("{head}.patch")).to_str().unwrap(),
2533                ],
2534            )
2535            .await
2536            .unwrap();
2537            assert_eq!(
2538                std::fs::read_to_string(recovered.join("source")).unwrap(),
2539                "cc05"
2540            );
2541            assert!(
2542                !std::fs::read(evidence.join(format!("{head}.patch")))
2543                    .unwrap()
2544                    .is_empty()
2545            );
2546            assert_eq!(
2547                std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
2548                "review findings"
2549            );
2550            if missing {
2551                assert_eq!(
2552                    std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
2553                    "original snapshot"
2554                );
2555            } else {
2556                assert!(!artifacts.join("cand-A.patch").exists());
2557            }
2558            assert!(run_dir.join("run.json").exists());
2559        }
2560    }
2561
2562    #[tokio::test]
2563    async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
2564        use std::os::unix::fs::PermissionsExt as _;
2565        let f = Fx::new().await;
2566        let source = f.run("dd03", true).await;
2567        let empty = f.run("dd04", false).await;
2568        let task = f.task(&[source, empty.clone()]);
2569        let entered = f.repo.join("proof-entered");
2570        let release = f.repo.join("proof-release");
2571        let hook = f.repo.join("proof-monitor");
2572        std::fs::write(&hook, format!(
2573            "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
2574            entered.display(), release.display())).unwrap();
2575        std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
2576        git::git(
2577            &f.repo,
2578            &["config", "core.fsmonitor", hook.to_str().unwrap()],
2579        )
2580        .await
2581        .unwrap();
2582        struct ReleaseProof(PathBuf);
2583        impl Drop for ReleaseProof {
2584            fn drop(&mut self) {
2585                let _ = std::fs::write(&self.0, "continue");
2586            }
2587        }
2588        let _release_on_panic = ReleaseProof(release.clone());
2589        let home = f.home.clone();
2590        let sweep = tokio::spawn(async move {
2591            let mut report = Report::default();
2592            retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
2593            report
2594        });
2595        tokio::time::timeout(std::time::Duration::from_secs(5), async {
2596            while !entered.exists() {
2597                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2598            }
2599        })
2600        .await
2601        .unwrap();
2602        let queue = crate::queue::Queue::at(f.home.join("queue"));
2603        let mut selected = queue.get(&task.id).unwrap();
2604        selected.resume_override = Some(crate::queue::OperatorResume {
2605            question_id: "during-slow-proof".into(),
2606            at: Fx::now(),
2607            conductor_rehold: None,
2608            forced: false,
2609            pinned_run: Some(empty.id.clone()),
2610        });
2611        let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
2612        let quick = match tokio::time::timeout(std::time::Duration::from_secs(2), &mut update).await
2613        {
2614            Ok(result) => {
2615                result.unwrap().unwrap();
2616                true
2617            }
2618            Err(_) => {
2619                std::fs::write(&release, "continue").unwrap();
2620                update.await.unwrap().unwrap();
2621                false
2622            }
2623        };
2624        std::fs::write(release, "continue").unwrap();
2625        let report = tokio::time::timeout(std::time::Duration::from_secs(8), sweep)
2626            .await
2627            .unwrap()
2628            .unwrap();
2629        assert!(
2630            quick,
2631            "task writers must not wait behind Git retirement proof"
2632        );
2633        assert_eq!(report.retired, 0);
2634        assert!(report.retained.iter().any(|r| r.contains("task changed")));
2635        assert!(empty.candidates[0].worktree.exists());
2636        assert!(
2637            RunState::load_under(&empty.id, &f.home)
2638                .unwrap()
2639                .retired_to
2640                .is_none()
2641        );
2642    }
2643
2644    #[tokio::test]
2645    async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
2646        let f = Fx::new().await;
2647        let source = f.run("dd01", true).await;
2648        let mut empty = f.run("dd02", false).await;
2649        let expected = f.task(&[source.clone(), empty.clone()]);
2650        let queue = crate::queue::Queue::at(f.home.join("queue"));
2651        let _claim = queue.claim(&expected.id).unwrap();
2652        // The proof holds run/output leases, never the task write lock.
2653        let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
2654        let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
2655        let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
2656        let mut selected = queue.get(&expected.id).unwrap();
2657        selected.resume_override = Some(crate::queue::OperatorResume {
2658            question_id: "new-owner-decision".into(),
2659            at: Fx::now(),
2660            conductor_rehold: None,
2661            forced: false,
2662            pinned_run: Some(empty.id.clone()),
2663        });
2664        queue.put(&mut selected).unwrap();
2665        let error = commit_retirement(
2666            &f.home,
2667            &queue,
2668            &expected,
2669            &mut empty,
2670            &source.id,
2671            &expected.runs,
2672            Fx::now(),
2673        )
2674        .unwrap_err();
2675        assert!(error.to_string().contains("task changed"));
2676        assert!(
2677            RunState::load_under(&empty.id, &f.home)
2678                .unwrap()
2679                .retired_to
2680                .is_none()
2681        );
2682        assert!(empty.candidates[0].worktree.exists());
2683        assert_eq!(
2684            queue
2685                .get(&expected.id)
2686                .unwrap()
2687                .resume_override
2688                .unwrap()
2689                .pinned_run,
2690            Some(empty.id.clone())
2691        );
2692        assert!(
2693            !f.home
2694                .join("recovery-lineages")
2695                .join(format!("{}.json", expected.id))
2696                .exists()
2697        );
2698        // Re-reading the new snapshot must not bypass its explicit pin either.
2699        assert!(
2700            commit_retirement(
2701                &f.home,
2702                &queue,
2703                &selected,
2704                &mut empty,
2705                &source.id,
2706                &selected.runs,
2707                Fx::now()
2708            )
2709            .unwrap_err()
2710            .to_string()
2711            .contains("selected this recovery source")
2712        );
2713    }
2714
2715    #[tokio::test]
2716    async fn unknown_targets_cargo_owners_and_driver_resumes_are_protected() {
2717        let f = Fx::new().await;
2718        let state = f.run("cccc", false).await;
2719        let wt = &state.candidates[0].worktree;
2720        Fx::target(wt);
2721        std::fs::write(wt.join("target/notes"), "user file").unwrap();
2722        assert!(cargo_target(wt).await.is_err());
2723        std::fs::remove_file(wt.join("target/notes")).unwrap();
2724        std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
2725        assert!(
2726            !declared_outputs(&wt.join("target"))
2727                .unwrap()
2728                .contains(&wt.join("target/debug/link"))
2729        );
2730        std::fs::remove_file(wt.join("target/debug/link")).unwrap();
2731        let cargo = cargo_locks(&wt.join("target")).unwrap();
2732        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2733        assert_eq!(out.freed_bytes, 0);
2734        assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
2735        drop(cargo);
2736        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2737        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2738        assert_eq!(out.freed_bytes, 0);
2739        drop(driver);
2740        let exclusive = OutputLease::take(wt, true).unwrap();
2741        assert!(
2742            OutputLease::borrow(wt).is_err(),
2743            "resume cannot start a writer during cleanup"
2744        );
2745        drop(exclusive);
2746        assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
2747    }
2748
2749    #[tokio::test]
2750    async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
2751        let f = Fx::new().await;
2752        let source = f.run("0000", true).await;
2753        let empty = f.run("0001", false).await;
2754        let changed = f.run("0002", false).await;
2755        let unknown = f.run("0003", false).await;
2756        let unproven = f.run("0004", false).await;
2757        f.task(&[
2758            source.clone(),
2759            empty.clone(),
2760            changed.clone(),
2761            unknown.clone(),
2762            unproven.clone(),
2763        ]);
2764        for run in [&empty, &changed, &unknown] {
2765            Fx::target(&run.candidates[0].worktree);
2766        }
2767        let reviewer = empty.worktree_root().join("review-1");
2768        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
2769            .await
2770            .unwrap();
2771        Fx::target(&reviewer);
2772        let mut disk = Fx::disk();
2773        disk.auto_fold = false;
2774        let report = sweep(&f.home, &disk, Fx::now()).await;
2775        assert_eq!(report.freed_bytes, 800, "{report:?}");
2776        let wt = &empty.candidates[0].worktree;
2777        assert!(wt.join("target/debug/output.d").exists());
2778        assert!(!wt.join("target/debug/output").exists());
2779        let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
2780        let proof: CargoResidue =
2781            serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
2782        assert_eq!(
2783            proof.files[Path::new("debug/output.d")],
2784            std::fs::read(wt.join("target/debug/output.d")).unwrap()
2785        );
2786        std::fs::write(
2787            changed.candidates[0].worktree.join("target/debug/output.d"),
2788            "owner notes",
2789        )
2790        .unwrap();
2791        std::fs::write(
2792            unknown.candidates[0].worktree.join("target/debug/notes"),
2793            "keep",
2794        )
2795        .unwrap();
2796        Fx::target(&unproven.candidates[0].worktree);
2797        std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
2798        disk.auto_fold = true;
2799        let locks = cargo_locks(&wt.join("target")).unwrap();
2800        let mut report = Report::default();
2801        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2802        assert_eq!(report.retired, 0, "{report:?}");
2803        assert!(
2804            report
2805                .retained
2806                .iter()
2807                .any(|reason| reason.contains("Cargo build"))
2808        );
2809        drop(locks);
2810        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2811        assert_eq!(report.retired, 1, "{report:?}");
2812        assert!(!wt.exists());
2813        assert!(!reviewer.exists());
2814        assert!(
2815            residue_proof(&f.home, &empty.id, &reviewer)
2816                .unwrap()
2817                .exists()
2818        );
2819        assert!(
2820            proof_path.exists(),
2821            "metadata evidence outlives the worktree"
2822        );
2823        assert_eq!(
2824            RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
2825            Some(source.id)
2826        );
2827        for run in [&changed, &unknown, &unproven] {
2828            assert!(run.candidates[0].worktree.exists());
2829            assert!(
2830                RunState::load_under(&run.id, &f.home)
2831                    .unwrap()
2832                    .retired_to
2833                    .is_none()
2834            );
2835        }
2836    }
2837
2838    #[tokio::test]
2839    async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
2840        let f = Fx::new().await;
2841        let source = f.run("0000", true).await;
2842        let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
2843            .await
2844            .unwrap();
2845        let mut runs = vec![source.clone()];
2846        for n in 1..=12 {
2847            runs.push(f.run(&format!("{n:04}"), false).await);
2848        }
2849        let dirty = runs[1].candidates[0].worktree.clone();
2850        std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
2851        let ignored = runs[2].candidates[0].worktree.clone();
2852        std::fs::create_dir(ignored.join("scratch")).unwrap();
2853        std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
2854        // A missing candidate array does not prove that an unrecorded worktree
2855        // is empty: discovery must inspect Git and the worktree itself.
2856        runs[1].candidates.clear();
2857        runs[1].save_under(&f.home).unwrap();
2858        let task = f.task(&runs);
2859        let mut total = 0;
2860        for _ in 0..4 {
2861            let mut report = Report::default();
2862            retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
2863            assert!(report.retired <= 5);
2864            total += report.retired;
2865        }
2866        assert_eq!(total, 10);
2867        assert!(dirty.exists());
2868        assert!(ignored.exists());
2869        assert_eq!(
2870            git::rev_parse(&f.repo, &source.candidates[0].branch)
2871                .await
2872                .unwrap(),
2873            source_head
2874        );
2875        let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
2876        assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
2877        assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
2878        let mut released = task.clone();
2879        released.release();
2880        assert_eq!(
2881            crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
2882                RunState::load_under(id, &f.home)
2883            }),
2884            Some(source.id.clone())
2885        );
2886
2887        assert!(
2888            f.home
2889                .join("runs")
2890                .join(&retired.id)
2891                .join("run.json")
2892                .exists()
2893        );
2894        assert!(
2895            f.home
2896                .join("recovery-lineages")
2897                .join(format!("{}.json", task.id))
2898                .exists()
2899        );
2900    }
2901
2902    #[tokio::test]
2903    async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
2904        let f = Fx::new().await;
2905        let a = f.run("dddd", true).await;
2906        let b = f.run("eeee", true).await;
2907        let empty = f.run("ffff", false).await;
2908        let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
2909        assert!(
2910            crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
2911            "unfinished queue history protects older source branches from ordinary folding"
2912        );
2913        let mut prior = a.clone();
2914        assert!(
2915            crate::graph::fold_run(&mut prior, true, &f.home)
2916                .await
2917                .unwrap_err()
2918                .to_string()
2919                .contains(&task.id)
2920        );
2921        let mut report = Report::default();
2922        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
2923        assert_eq!(report.retired, 0);
2924        assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
2925        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2926        Fx::target(&a.candidates[0].worktree);
2927        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2928        assert!(
2929            report.freed_bytes > 0,
2930            "source retention does not retain regenerable outputs"
2931        );
2932        assert!(a.candidates[0].worktree.exists());
2933        assert!(b.candidates[0].worktree.exists());
2934        assert!(empty.candidates[0].worktree.exists());
2935    }
2936
2937    #[tokio::test]
2938    async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
2939        let f = Fx::new().await;
2940        let authority = f.run("p000", true).await;
2941        let question_run = f.run("p001", false).await;
2942        let mut checkpoint = f.run("p002", false).await;
2943        checkpoint.land_approval = Some(crate::run::LandApproval {
2944            question: "pending".into(),
2945            head: f.base.clone(),
2946        });
2947        checkpoint.save_under(&f.home).unwrap();
2948        let retained = f.run("p003", false).await;
2949        std::fs::write(
2950            f.home
2951                .join("runs")
2952                .join(&retained.id)
2953                .join("retain-recovery-sources"),
2954            "owner",
2955        )
2956        .unwrap();
2957        let mut partial = f.run("p004", false).await;
2958        let reviewer = partial.worktree_root().join("review-1");
2959        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
2960            .await
2961            .unwrap();
2962        // Model a crash after the retirement fence but only one removal.
2963        partial.retired_to = Some(authority.id.clone());
2964        partial.save_under(&f.home).unwrap();
2965        assert!(
2966            git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
2967                .await
2968                .unwrap()
2969        );
2970        let git_checkpoint = f.run("p005", false).await;
2971        let location = git::git(
2972            &git_checkpoint.candidates[0].worktree,
2973            &[
2974                "rev-parse",
2975                "--path-format=absolute",
2976                "--git-path",
2977                "sequencer",
2978            ],
2979        )
2980        .await
2981        .unwrap();
2982        std::fs::create_dir(location.trim()).unwrap();
2983        let mut preservation = f.run("p006", false).await;
2984        preservation
2985            .candidate_preservation
2986            .insert('A', "pending preservation".into());
2987        preservation.save_under(&f.home).unwrap();
2988        let task = f.task(&[
2989            preservation.clone(),
2990            git_checkpoint.clone(),
2991            authority.clone(),
2992            question_run.clone(),
2993            checkpoint.clone(),
2994            retained.clone(),
2995            partial.clone(),
2996        ]);
2997        let questions = crate::ask::Questions::at(f.home.join("questions"));
2998        let mut question = crate::ask::Question::new(
2999            question_run.id.clone(),
3000            "implement".into(),
3001            "fixture".into(),
3002            "owner decision".into(),
3003            String::new(),
3004            vec!["keep".into()],
3005        );
3006        questions.put(&mut question).unwrap();
3007        let mut report = Report::default();
3008        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3009        assert_eq!(report.retired, 1, "{report:?}");
3010        assert!(!reviewer.exists());
3011        for state in [
3012            &question_run,
3013            &checkpoint,
3014            &retained,
3015            &git_checkpoint,
3016            &preservation,
3017        ] {
3018            assert!(state.candidates[0].worktree.exists());
3019            assert!(
3020                RunState::load_under(&state.id, &f.home)
3021                    .unwrap()
3022                    .retired_to
3023                    .is_none()
3024            );
3025        }
3026        assert!(questions.get(&question.id).unwrap().status.open());
3027        assert_eq!(
3028            crate::queue::Queue::at(f.home.join("queue"))
3029                .get(&task.id)
3030                .unwrap()
3031                .status,
3032            task.status
3033        );
3034    }
3035
3036    #[tokio::test]
3037    async fn preservation_schema_migration_remains_distinct_from_retirement() {
3038        let f = Fx::new().await;
3039        let mut state = f.run("p007", false).await;
3040        state.schema = 15;
3041        state.save_under(&f.home).unwrap();
3042        let native = RunState::load_under(&state.id, &f.home).unwrap();
3043        assert_eq!(native.schema, crate::run::SCHEMA);
3044        assert!(native.candidate_preservation.is_empty());
3045        assert!(native.retired_to.is_none());
3046        state.schema = 14;
3047        state.save_under(&f.home).unwrap();
3048        let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
3049        assert!(legacy.candidate_preservation.contains_key(&'A'));
3050        legacy.save_under(&f.home).unwrap();
3051        let restored = RunState::load_under(&state.id, &f.home).unwrap();
3052        assert_eq!(
3053            restored.candidate_preservation,
3054            legacy.candidate_preservation
3055        );
3056    }
3057
3058    #[tokio::test]
3059    async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
3060        let f = Fx::new().await;
3061        let cache = f.repo.join("cache");
3062        let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
3063        std::fs::create_dir_all(&cache).unwrap();
3064        assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3065        assert!(
3066            crate::cache::maintenance_prune(&f.home, &cache, 1)
3067                .unwrap()
3068                .is_none()
3069        );
3070        drop(guard);
3071        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3072    }
3073
3074    #[tokio::test]
3075    async fn launcher_death_does_not_release_live_tool_output_reservations() {
3076        let f = Fx::new().await;
3077        let run = f.run("abcd", false).await;
3078        let wt = &run.candidates[0].worktree;
3079        let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
3080        helper
3081            .quiet()
3082            .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
3083            .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
3084            .env("MAGI_TEST_WORKTREE", wt)
3085            .stdin(std::process::Stdio::null())
3086            .stdout(std::process::Stdio::null())
3087            .stderr(std::process::Stdio::null())
3088            .kill_on_drop(true);
3089        assert!(
3090            tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
3091                .await
3092                .unwrap()
3093                .unwrap()
3094                .success()
3095        );
3096        let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
3097            .unwrap()
3098            .trim()
3099            .parse()
3100            .unwrap();
3101        struct KillTool(i32);
3102        impl Drop for KillTool {
3103            fn drop(&mut self) {
3104                // SAFETY: PID belongs to the bounded helper above.
3105                unsafe { libc::kill(self.0, libc::SIGKILL) };
3106            }
3107        }
3108        let tool = KillTool(pid);
3109        assert!(OutputLease::take(wt, true).is_err());
3110        assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
3111        drop(tool);
3112        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3113            loop {
3114                if OutputLease::take(wt, true).is_ok()
3115                    && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
3116                {
3117                    break;
3118                }
3119                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3120            }
3121        })
3122        .await
3123        .unwrap();
3124    }
3125
3126    #[tokio::test]
3127    async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
3128        let f = Fx::new().await;
3129        let run = f.run("abcd", false).await;
3130        let wt = &run.candidates[0].worktree;
3131        let cache = f.repo.join("cache");
3132        std::fs::create_dir(&cache).unwrap();
3133        for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
3134            let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
3135            let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
3136            cmd.quiet()
3137                .args(["--exact", "seat_process::tests::orphan_tool_helper"])
3138                .env("MAGI_TEST_ORPHAN_PID", &pid_file)
3139                .stdin(std::process::Stdio::null())
3140                .stdout(std::process::Stdio::null())
3141                .stderr(std::process::Stdio::null())
3142                .kill_on_drop(true);
3143            if clear_owner {
3144                cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
3145            } else {
3146                cmd.env("MAGI_TEST_CLOSE_FDS", "1");
3147            }
3148            let mut running = crate::seat_process::Running::spawn(
3149                cmd,
3150                None,
3151                vec![
3152                    OutputLease::borrow(wt).unwrap().unwrap(),
3153                    OutputLease::cache(&f.home, &cache, false).unwrap(),
3154                ],
3155            )
3156            .unwrap();
3157            assert!(
3158                tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
3159                    .await
3160                    .unwrap()
3161                    .unwrap()
3162                    .success()
3163            );
3164            let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
3165            struct KillTool(Option<i32>);
3166            impl Drop for KillTool {
3167                fn drop(&mut self) {
3168                    // SAFETY: PID from our helper; kill also bounds panic paths.
3169                    if let Some(pid) = self.0 {
3170                        unsafe { libc::kill(pid, libc::SIGKILL) };
3171                    }
3172                }
3173            }
3174            let mut tool = KillTool(Some(pid));
3175            // The CLI has already exited and cannot supply a parent-tree
3176            // relationship. Either the witness or inherited environment must
3177            // retain ownership; test each independently.
3178            assert!(OutputLease::take(wt, true).is_err());
3179            let finish = if cancel {
3180                drop(running);
3181                None
3182            } else {
3183                Some(tokio::spawn(async move { running.finish().await }))
3184            };
3185            if clear_owner {
3186                tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3187                assert!(crate::proc::pid_alive(pid as u32));
3188                assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
3189                assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3190                assert!(
3191                    crate::cache::maintenance_prune(&f.home, &cache, 0)
3192                        .unwrap()
3193                        .is_none()
3194                );
3195                if let Some(finish) = &finish {
3196                    assert!(!finish.is_finished());
3197                }
3198                // Witness-only ownership waits for natural/external tool exit.
3199                unsafe { libc::kill(pid, libc::SIGKILL) };
3200            }
3201            if let Some(finish) = finish {
3202                tokio::time::timeout(std::time::Duration::from_secs(5), finish)
3203                    .await
3204                    .unwrap()
3205                    .unwrap();
3206            }
3207            tokio::time::timeout(std::time::Duration::from_secs(5), async {
3208                loop {
3209                    if OutputLease::take(wt, true).is_ok()
3210                        && OutputLease::cache(&f.home, &cache, true).is_ok()
3211                    {
3212                        // Token-owned orphans must be stopped by cleanup itself.
3213                        if !clear_owner {
3214                            assert!(!crate::proc::pid_alive(pid as u32));
3215                        }
3216                        break;
3217                    }
3218                    tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3219                }
3220            })
3221            .await
3222            .unwrap();
3223            tool.0 = None;
3224        }
3225    }
3226
3227    #[tokio::test]
3228    async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
3229        let f = Fx::new().await;
3230        let run = f.run("abcd", false).await;
3231        let wt = &run.candidates[0].worktree;
3232        let cache = f.repo.join("cache");
3233        std::fs::create_dir(&cache).unwrap();
3234        let pid_file = f.repo.join("child-pid");
3235        let mut cmd = tokio::process::Command::new("sh");
3236        cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
3237            .quiet()
3238            .env("CHILD_PID_FILE", &pid_file)
3239            .process_group(0)
3240            .kill_on_drop(true);
3241        let running = crate::seat_process::Running::spawn(
3242            cmd,
3243            None,
3244            vec![
3245                OutputLease::borrow(wt).unwrap().unwrap(),
3246                OutputLease::cache(&f.home, &cache, false).unwrap(),
3247            ],
3248        )
3249        .unwrap();
3250        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3251            while !pid_file.exists() {
3252                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3253            }
3254        })
3255        .await
3256        .unwrap();
3257        assert!(OutputLease::take(wt, true).is_err());
3258        assert!(
3259            crate::cache::maintenance_prune(&f.home, &cache, 0)
3260                .unwrap()
3261                .is_none()
3262        );
3263        let pid: u32 = std::fs::read_to_string(pid_file)
3264            .unwrap()
3265            .trim()
3266            .parse()
3267            .unwrap();
3268        drop(running);
3269        tokio::time::timeout(std::time::Duration::from_secs(10), async {
3270            loop {
3271                if let Ok(guard) = OutputLease::take(wt, true) {
3272                    assert!(
3273                        !crate::proc::pid_alive(pid),
3274                        "output lock released before tool death"
3275                    );
3276                    drop(guard);
3277                    break;
3278                }
3279                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3280            }
3281        })
3282        .await
3283        .unwrap();
3284        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3285    }
3286
3287    #[tokio::test]
3288    async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
3289        let f = Fx::new().await;
3290        let reason = tokio::time::timeout(
3291            std::time::Duration::from_secs(2),
3292            emergency(vec![f.repo.clone()], u64::MAX),
3293        )
3294        .await
3295        .unwrap();
3296        assert!(reason.contains("disk pressure"));
3297        assert!(
3298            tokio::time::timeout(
3299                std::time::Duration::from_millis(10),
3300                emergency(vec![f.repo.clone()], 0)
3301            )
3302            .await
3303            .is_err()
3304        );
3305        assert_eq!(
3306            std::fs::read_to_string(f.repo.join("source")).unwrap(),
3307            "base\n"
3308        );
3309    }
3310
3311    #[tokio::test]
3312    async fn automatic_requeues_cannot_refund_failed_input_generation() {
3313        let f = Fx::new().await;
3314        let mut run = f.run("1234", false).await;
3315        run.prep_failure = Some("cherry-pick conflict".into());
3316        let mut task = f.task(std::slice::from_ref(&run));
3317        let inputs = crate::queue::recovery_inputs(&run.sources);
3318        for _ in 0..108 {
3319            task.recovery_failed(&run);
3320            task.requeue();
3321            assert_eq!(task.attempts, 0);
3322        }
3323        let bytes = serde_json::to_vec(&task).unwrap();
3324        let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
3325        assert!(
3326            restored
3327                .recovery_exhausted(&run.base_commit, &inputs, 2)
3328                .is_some()
3329        );
3330        assert!(
3331            restored
3332                .recovery_exhausted("different-base", &inputs, 2)
3333                .is_none()
3334        );
3335        let original_base = run.base_commit.clone();
3336        run.base_commit = "different-base".into();
3337        restored.recovery_failed(&run);
3338        assert!(
3339            restored
3340                .recovery_exhausted(&original_base, &inputs, 2)
3341                .is_some(),
3342            "returning to old inputs does not replenish their budget"
3343        );
3344        assert!(
3345            restored
3346                .recovery_exhausted(&run.base_commit, &inputs, 2)
3347                .is_none()
3348        );
3349        restored.release_operator("test-owner", None);
3350        assert!(
3351            restored
3352                .recovery_exhausted(&run.base_commit, &inputs, 2)
3353                .is_none()
3354        );
3355    }
3356}