Skip to main content

magi/
retention.rs

1//! Bounded reclamation of retained attempts and Cargo outputs. Source recovery
2//! and build eviction have separate authority: evicting a target never retires a run.
3pub mod reconcile;
4
5use crate::{
6    git,
7    proc::Quiet as _,
8    run::{Liveness, RunState},
9};
10use anyhow::{Context, Result, ensure};
11use jiff::Timestamp;
12use serde::{Deserialize, Serialize};
13use std::{
14    fs::{File, OpenOptions},
15    path::{Path, PathBuf},
16};
17
18/// An OS lease on a worktree's outputs, independent of repository coordination.
19/// Kept by the process cleanup owner until descendants terminate, including
20/// reparented tools retaining the inherited liveness witness.
21#[derive(Debug)]
22pub(crate) struct OutputLease {
23    _file: File,
24}
25
26impl OutputLease {
27    #[cfg(unix)]
28    pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
29        use std::os::fd::AsRawFd;
30        self._file.as_raw_fd()
31    }
32
33    pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
34        if !worktree.join(".git").is_file() {
35            return Ok(None);
36        }
37        Self::take(worktree, false).map(Some)
38    }
39    pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
40        use std::hash::{Hash, Hasher};
41        let mut hash = std::collections::hash_map::DefaultHasher::new();
42        let mut normalized = cache.to_owned();
43        let mut missing = Vec::new();
44        while !normalized.exists() {
45            let Some(name) = normalized.file_name() else {
46                break;
47            };
48            missing.push(name.to_owned());
49            if !normalized.pop() {
50                break;
51            }
52        }
53        normalized = normalized.canonicalize().unwrap_or(normalized);
54        for part in missing.into_iter().rev() {
55            normalized.push(part);
56        }
57        normalized.hash(&mut hash);
58        let dir = home.join("cache-leases");
59        std::fs::create_dir_all(&dir)?;
60        Self::lock(
61            &dir.join(format!("outputs-{:x}.lock", hash.finish())),
62            exclusive,
63        )
64    }
65    pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
66        let out = std::process::Command::new("git")
67            .args(["rev-parse", "--absolute-git-dir"])
68            .current_dir(worktree)
69            .quiet()
70            .output()?;
71        ensure!(
72            out.status.success(),
73            "cannot identify output lease directory"
74        );
75        let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
76        Self::lock(&dir.join("magi-output.lock"), exclusive)
77    }
78    fn lock(path: &Path, exclusive: bool) -> Result<Self> {
79        let file = OpenOptions::new()
80            .create(true)
81            .truncate(false)
82            .read(true)
83            .write(true)
84            .open(path)?;
85        #[cfg(unix)]
86        {
87            use std::os::fd::AsRawFd;
88            let mode = if exclusive {
89                libc::LOCK_EX
90            } else {
91                libc::LOCK_SH
92            };
93            // SAFETY: live file descriptor, no retained pointers.
94            ensure!(
95                unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
96                "worktree outputs are in use or being reclaimed"
97            );
98        }
99        #[cfg(not(unix))]
100        let _ = exclusive;
101        Ok(Self { _file: file })
102    }
103}
104#[cfg(unix)]
105impl Drop for OutputLease {
106    fn drop(&mut self) {
107        use std::os::fd::AsRawFd;
108        // SAFETY: unlock this guard's live descriptor, including inherited copies.
109        unsafe {
110            libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
111        }
112    }
113}
114
115#[derive(Default, Debug, Serialize, Deserialize)]
116#[serde(default)]
117pub(crate) struct Report {
118    pub checked_at: Option<Timestamp>,
119    pub limit_bytes: u64,
120    pub reclaim_budget_bytes: u64,
121    pub measured_bytes: u64,
122    pub freed_bytes: u64,
123    pub retired: usize,
124    pub retained: Vec<String>,
125    pub failures: Vec<String>,
126}
127
128fn plain_dir(path: &Path) -> bool {
129    std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
130}
131
132/// Old run records outlive their checkout and output directories. Only an
133/// absent path means nothing to reclaim; inaccessible paths remain errors.
134fn run_paths_missing(state: &RunState) -> Result<bool> {
135    for path in [state.worktree_root(), state.repo.clone()] {
136        match std::fs::metadata(&path) {
137            Ok(_) => {}
138            Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
139            Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
140        }
141    }
142    Ok(false)
143}
144
145/// Enumerate actual registered worktrees, not just candidates in run.json.
146async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
147    let root = state.worktree_root();
148    if !plain_dir(&root) {
149        return Ok(Vec::new());
150    }
151    let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
152    registered_worktrees(state, &list)
153}
154
155fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
156    let root = state.worktree_root();
157    if !plain_dir(&root) {
158        return Ok(Vec::new());
159    }
160    let root = root.canonicalize()?;
161    let mut paths = Vec::new();
162    for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
163        let path = PathBuf::from(path);
164        if plain_dir(&path)
165            && path.parent() == Some(root.as_path())
166            && path.canonicalize()? == path
167            && path.join(".git").is_file()
168        {
169            paths.push(path);
170        }
171    }
172    Ok(paths)
173}
174
175/// Recognize Cargo's generated directory, never merely an ignored directory.
176/// Unknown root entries and symlinked cache identity/profile paths fail closed.
177async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
178    let target = worktree.join("target");
179    ensure!(plain_dir(&target), "target absent or not a plain directory");
180    ensure!(target.canonicalize()? == target, "target path changed");
181    ensure!(
182        git::git(worktree, &["ls-files", "--", "target"])
183            .await?
184            .is_empty(),
185        "tracked target contents"
186    );
187    ensure!(
188        git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
189            .await?
190            .ok(),
191        "target is not ignored"
192    );
193    for name in ["CACHEDIR.TAG", ".rustc_info.json"] {
194        let path = target.join(name);
195        ensure!(
196            std::fs::symlink_metadata(&path)?.is_file() && path.canonicalize()? == path,
197            "cache identity is not a plain file"
198        );
199    }
200    let tag = std::fs::read_to_string(target.join("CACHEDIR.TAG"))?;
201    ensure!(
202        tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55d"),
203        "unknown Cargo cache tag"
204    );
205    let info: serde_json::Value =
206        serde_json::from_slice(&std::fs::read(target.join(".rustc_info.json"))?)?;
207    ensure!(
208        info.get("rustc_fingerprint").is_some(),
209        "unknown Cargo identity"
210    );
211    for entry in std::fs::read_dir(&target)? {
212        let entry = entry?;
213        let name = entry.file_name();
214        let name = name.to_str().context("non-UTF8 output")?;
215        ensure!(
216            matches!(
217                name,
218                "debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json"
219            ),
220            "unrecognized target entry {name}"
221        );
222        if matches!(name, "debug" | "release") {
223            ensure!(
224                plain_dir(&entry.path()) && entry.path().canonicalize()? == entry.path(),
225                "profile is not a plain directory"
226            );
227        }
228    }
229    Ok(target)
230}
231
232fn artifact_signature(path: &Path) -> Result<bool> {
233    use std::io::Read as _;
234    let mut header = [0; 8];
235    let n = File::open(path)?.read(&mut header)?;
236    let header = &header[..n];
237    Ok(header.starts_with(b"\x7fELF")
238        || header.starts_with(b"!<arch>\n")
239        || header.starts_with(b"rust\0\0\0")
240        || header.starts_with(b"MZ\x90\0")
241        || [
242            [0xfe, 0xed, 0xfa, 0xce],
243            [0xce, 0xfa, 0xed, 0xfe],
244            [0xfe, 0xed, 0xfa, 0xcf],
245            [0xcf, 0xfa, 0xed, 0xfe],
246            [0xca, 0xfe, 0xba, 0xbe],
247            [0xca, 0xfe, 0xba, 0xbf],
248        ]
249        .iter()
250        .any(|magic| header.starts_with(magic)))
251}
252
253/// Legacy tags identify a cache, not ownership of every file inside it.
254/// Only direct rustc outputs named by dep-info may be reclaimed. Build-script
255/// output, incremental caches, unknown files and the manifests themselves stay.
256fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
257    let mut outputs = std::collections::BTreeSet::new();
258    for profile in ["debug", "release"] {
259        for dir in [target.join(profile), target.join(profile).join("deps")] {
260            if !plain_dir(&dir) || dir.canonicalize()? != dir {
261                continue;
262            }
263            for entry in std::fs::read_dir(&dir)? {
264                let path = entry?.path();
265                if path.extension().is_none_or(|ext| ext != "d")
266                    || !std::fs::symlink_metadata(&path)?.is_file()
267                    || std::fs::metadata(&path)?.len() > 1024 * 1024
268                    || path.canonicalize()? != path
269                {
270                    continue;
271                }
272                let body = std::fs::read_to_string(&path)?;
273                for line in body.lines() {
274                    let Some((names, dependencies)) = line.split_once(": ") else {
275                        continue;
276                    };
277                    if dependencies.trim().is_empty() {
278                        continue;
279                    }
280                    // Rustc escapes spaces and backslashes in Make paths.
281                    // Unsupported Make syntax fails closed for that record.
282                    let mut words = Vec::new();
283                    let mut word = String::new();
284                    let mut escaped = false;
285                    for c in names.chars() {
286                        if escaped {
287                            word.push(c);
288                            escaped = false;
289                        } else if c == '\\' {
290                            escaped = true;
291                        } else if c.is_whitespace() {
292                            if !word.is_empty() {
293                                words.push(std::mem::take(&mut word));
294                            }
295                        } else {
296                            word.push(c);
297                        }
298                    }
299                    if escaped || word.contains(['$', '%']) {
300                        continue;
301                    }
302                    if !word.is_empty() {
303                        words.push(word);
304                    }
305                    for name in words {
306                        let output = PathBuf::from(name);
307                        let stem = path.file_stem().unwrap_or_default().to_string_lossy();
308                        let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
309                        let artifact_name =
310                            output_stem == stem || output_stem == format!("lib{stem}");
311                        let artifact_kind = output.extension().is_none_or(|ext| {
312                            matches!(
313                                ext.to_str(),
314                                Some(
315                                    "rlib"
316                                        | "rmeta"
317                                        | "so"
318                                        | "dylib"
319                                        | "a"
320                                        | "lib"
321                                        | "dll"
322                                        | "exe"
323                                        | "pdb"
324                                )
325                            )
326                        });
327                        if artifact_name
328                            && artifact_kind
329                            && output.parent() == Some(dir.as_path())
330                            && output.extension().is_none_or(|ext| ext != "d")
331                            && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
332                            && output.canonicalize().is_ok_and(|p| p == output)
333                            && artifact_signature(&output)?
334                        {
335                            outputs.insert(output);
336                        }
337                    }
338                }
339            }
340        }
341    }
342    Ok(outputs.into_iter().collect())
343}
344
345#[derive(Serialize, Deserialize)]
346struct CargoResidue {
347    target: PathBuf,
348    files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
349}
350
351// Keep the exact metadata as historical evidence. An old proof cannot adopt
352// newly added files, changed metadata, symlinks or a different worktree.
353fn residue_files(
354    target: &Path,
355    outputs: &[PathBuf],
356) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
357    let mut files = std::collections::BTreeMap::new();
358    let mut dirs = vec![target.to_owned()];
359    let mut bytes = 0usize;
360    let mut entries = 0usize;
361    while let Some(dir) = dirs.pop() {
362        for entry in std::fs::read_dir(dir)? {
363            let path = entry?.path();
364            entries += 1;
365            ensure!(entries <= 4096, "residue inventory too large");
366            let metadata = std::fs::symlink_metadata(&path)?;
367            ensure!(
368                !metadata.file_type().is_symlink(),
369                "symlink in Cargo residue"
370            );
371            if metadata.is_dir() {
372                dirs.push(path);
373            } else {
374                ensure!(metadata.is_file(), "non-file in Cargo residue");
375                if outputs.contains(&path) {
376                    continue;
377                }
378                ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
379                let body = std::fs::read(&path)?;
380                bytes = bytes.saturating_add(body.len());
381                ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
382                files.insert(path.strip_prefix(target)?.to_owned(), body);
383            }
384        }
385    }
386    Ok(files)
387}
388
389fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
390    Ok(home
391        .join("runs")
392        .join(id)
393        .join("build-residue")
394        .join(worktree.file_name().context("missing worktree name")?))
395}
396
397fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
398    let files = residue_files(target, outputs)?;
399    for (relative, body) in &files {
400        let path = target.join(relative);
401        let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
402        let lock = matches!(
403            relative.to_str(),
404            Some("debug/.cargo-lock" | "release/.cargo-lock")
405        ) && body.is_empty();
406        let dep_info = path.extension().is_some_and(|ext| ext == "d")
407            && outputs.iter().any(|output| {
408                output.parent() == path.parent()
409                    && (output.file_stem() == path.file_stem()
410                        || output.file_stem().is_some_and(|stem| {
411                            stem.to_string_lossy()
412                                == format!(
413                                    "lib{}",
414                                    path.file_stem().unwrap_or_default().to_string_lossy()
415                                )
416                        }))
417            });
418        ensure!(identity || lock || dep_info, "unproven Cargo residue");
419    }
420    let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
421    std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
422    let tmp = proof.with_extension("tmp");
423    std::fs::write(
424        &tmp,
425        serde_json::to_vec(&CargoResidue {
426            target: target.to_owned(),
427            files,
428        })?,
429    )?;
430    std::fs::rename(tmp, proof)?;
431    Ok(())
432}
433
434pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
435    let target = cargo_target(worktree).await?;
436    let locks = cargo_locks(&target)?;
437    let proof: CargoResidue =
438        serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
439    ensure!(
440        proof.target == target && proof.files == residue_files(&target, &[])?,
441        "Cargo residue changed or contains unproven files"
442    );
443    Ok(locks)
444}
445
446/// Supplemental exclusion for Cargo builds launched outside the managed seats.
447pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
448    let mut held = Vec::new();
449    for profile in ["debug", "release"] {
450        let lock = target.join(profile).join(".cargo-lock");
451        if lock.exists() {
452            ensure!(
453                std::fs::symlink_metadata(&lock)?.is_file() && lock.canonicalize()? == lock,
454                "Cargo lock is not a plain file"
455            );
456            let file = OpenOptions::new().read(true).write(true).open(lock)?;
457            crate::pr_head::try_lock_file(&file).context("Cargo build is active")?;
458            held.push(file);
459        }
460    }
461    Ok(held)
462}
463
464/// All repositories share one cap and one per-pass deletion budget. The report
465/// survives a restart and explains every skipped run/target or failed operation.
466pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
467    let home = home.to_owned();
468    let disk = disk.clone();
469    let runtime = tokio::runtime::Handle::current();
470    tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
471        .await
472        .unwrap_or_else(|e| Report {
473            failures: vec![format!("retention worker failed: {e}")],
474            ..Report::default()
475        })
476}
477
478#[derive(Default, Serialize, Deserialize)]
479struct Measurement {
480    at: Option<Timestamp>,
481    bytes: u64,
482}
483
484async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
485    let mut report = Report {
486        checked_at: Some(now),
487        limit_bytes: disk.worktree_limit_bytes,
488        reclaim_budget_bytes: disk.worktree_reclaim_bytes,
489        ..Report::default()
490    };
491    if !cfg!(unix) {
492        report
493            .failures
494            .push("Automatic output reclamation requires Unix process containment".into());
495        return report;
496    }
497    let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
498        return report;
499    };
500    let mut targets = Vec::new();
501    let mut registrations = std::collections::HashMap::new();
502    let measurements_path = home.join("output-measurements.json");
503    let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
504        std::fs::read(&measurements_path)
505            .ok()
506            .and_then(|b| serde_json::from_slice(&b).ok())
507            .unwrap_or_default();
508    let mut observed = std::collections::BTreeSet::new();
509    let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
510        .into_iter()
511        .flatten()
512        .flatten()
513        .map(|e| e.file_name().to_string_lossy().into_owned())
514        .collect();
515    ids.sort();
516    for id in ids {
517        if !crate::run::is_run_id(&id) {
518            continue;
519        }
520        let state = match RunState::load_under(&id, home) {
521            Ok(state) => state,
522            Err(e) => {
523                report
524                    .retained
525                    .push(format!("{id}: unreadable run ownership: {e:#}"));
526                continue;
527            }
528        };
529        if disk.worktree_limit_bytes == 0 {
530            continue;
531        }
532        match run_paths_missing(&state) {
533            Ok(true) => continue,
534            Ok(false) => {}
535            Err(e) => {
536                report.failures.push(format!("{id}: {e:#}"));
537                continue;
538            }
539        }
540        let list = match registrations.entry(state.repo.clone()) {
541            std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
542            std::collections::hash_map::Entry::Vacant(entry) => {
543                match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
544                    Ok(list) => entry.insert(list),
545                    Err(e) => {
546                        // The checkout may disappear after the inventory check.
547                        // Do not confuse a missing Git executable with that case.
548                        if matches!(run_paths_missing(&state), Ok(true)) {
549                            continue;
550                        }
551                        report.failures.push(format!("{id}: {e:#}"));
552                        continue;
553                    }
554                }
555            }
556        };
557        let paths = match registered_worktrees(&state, list) {
558            Ok(paths) => paths,
559            Err(e) => {
560                report.failures.push(format!("{id}: {e:#}"));
561                continue;
562            }
563        };
564        for path in paths {
565            if !path.join("target").exists() {
566                continue;
567            }
568            let target = path.join("target");
569            observed.insert(target.clone());
570            let pressure =
571                crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
572            let measurement = measurements.entry(target).or_default();
573            if pressure
574                || measurement
575                    .at
576                    .is_none_or(|at| crate::clean::due(now, at, 300))
577            {
578                measurement.bytes = crate::disk::dir_size(&path.join("target"));
579                measurement.at = Some(now);
580            }
581            let size = measurement.bytes;
582            report.measured_bytes = report.measured_bytes.saturating_add(size);
583            targets.push((state.updated_at, id.clone(), path, size));
584        }
585    }
586    targets.sort_by_key(|t| t.0);
587    let mut remaining = report.measured_bytes;
588    for (updated, id, path, size) in targets {
589        if disk.worktree_limit_bytes == 0
590            || remaining <= disk.worktree_limit_bytes
591                && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
592        {
593            report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
594            continue;
595        }
596        if report.freed_bytes.saturating_add(size) > disk.worktree_reclaim_bytes {
597            report.retained.push(format!(
598                "{id}: target exceeds remaining per-pass byte budget"
599            ));
600            continue;
601        }
602        let result = async {
603            ensure!(
604                crate::clean::due(now, updated, disk.fold_grace_secs),
605                "within grace period"
606            );
607            let _owner = crate::pr_head::owner_lock(home, &id)?;
608            let state = RunState::load_under(&id, home)?;
609            ensure!(
610                state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
611                    == Liveness::Dead,
612                "run live or ownership unknown"
613            );
614            ensure!(crate::clean::due(now, state.updated_at, disk.fold_grace_secs), "recently used after inventory");
615            let _driver = crate::driver::maintenance(home, &id)?;
616            let _outputs = OutputLease::take(&path, true)?;
617            ensure!(
618                worktrees(&state).await?.contains(&path),
619                "worktree registration changed"
620            );
621            let target = cargo_target(&path).await?;
622            let _cargo = cargo_locks(&target)?;
623            let outputs = declared_outputs(&target)?;
624            ensure!(!outputs.is_empty(), "no dep-info-proven outputs; unknown files retained");
625            let current_size = outputs.iter().try_fold(0u64, |bytes, p| {
626                Ok::<_, std::io::Error>(bytes.saturating_add(std::fs::metadata(p)?.len()))
627            })?;
628            ensure!(report.freed_bytes.saturating_add(current_size) <= disk.worktree_reclaim_bytes,
629                "target grew beyond remaining per-pass budget");
630            // Record adoption before deletion; interrupted deletion is revalidated
631            // on the next pass. No source file or branch is modified.
632            let ledger = home.join("runs").join(&id).join("build-outputs.json");
633            std::fs::write(
634                ledger,
635                serde_json::to_vec(
636                    &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","files":outputs}),
637                )?,
638            )?;
639            // Unknown leftovers still permit artifact-only eviction, but never
640            // authorize retirement. Only a complete proven residue is adopted.
641            if let Err(error) = record_residue(home, &id, &target, &outputs) {
642                report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
643            }
644            for output in &outputs {
645                std::fs::remove_file(output)?;
646            }
647            measurements.remove(&target);
648            report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
649            Ok::<_, anyhow::Error>(current_size)
650        }
651        .await;
652        match result {
653            Ok(freed) => {
654                report.freed_bytes = report.freed_bytes.saturating_add(freed);
655                remaining = remaining.saturating_sub(freed);
656            }
657            Err(e) => report.retained.push(format!(
658                "{id}/{}: {e:#}",
659                path.file_name().unwrap_or_default().to_string_lossy()
660            )),
661        }
662    }
663    measurements.retain(|path, _| observed.contains(path));
664    if let Ok(body) = serde_json::to_vec(&measurements)
665        && let Err(e) = std::fs::write(&measurements_path, body)
666    {
667        report.failures.push(format!("output measurements: {e}"));
668    }
669    retire_empty(home, disk, now, &mut report).await;
670    if !report.failures.is_empty()
671        || disk.worktree_limit_bytes > 0
672            && remaining > disk.worktree_limit_bytes
673            && report.freed_bytes == 0
674    {
675        crate::notices::raise_in(
676            home,
677            crate::notices::Notice::warn(
678                "disk:retention",
679                "Retained build outputs could not be brought under the disk cap. Inspect the cleanup report for active owners, retained recovery sources and failures.",
680            ),
681        );
682    }
683    let store = home.join("retention.json");
684    if let Ok(body) = serde_json::to_vec_pretty(&report) {
685        let tmp = home.join("retention.json.tmp");
686        if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
687            tracing::warn!("retention report could not be saved: {e}");
688        }
689    }
690    tracing::info!(
691        measured_bytes = report.measured_bytes,
692        freed_bytes = report.freed_bytes,
693        retired = report.retired,
694        retained = report.retained.len(),
695        failures = report.failures.len(),
696        "retained worktree cleanup"
697    );
698    for reason in report.retained.iter().chain(&report.failures) {
699        tracing::debug!(%reason, "worktree retained");
700    }
701    report
702}
703
704async fn retire_empty(
705    home: &Path,
706    disk: &crate::config::Disk,
707    now: Timestamp,
708    report: &mut Report,
709) {
710    if !disk.auto_fold {
711        return;
712    }
713    let queue = crate::queue::Queue::at(home.join("queue"));
714    let questions = crate::ask::Questions::at(home.join("questions"));
715    let open = questions.list();
716    for listed in queue.list() {
717        let Ok(_claim) = queue.claim(&listed.id) else {
718            continue;
719        };
720        let Ok(task) = queue.get(&listed.id) else {
721            continue;
722        };
723        if open
724            .iter()
725            .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
726        {
727            report.retained.push(format!(
728                "{}: task has an unanswered owner decision",
729                task.id
730            ));
731            continue;
732        }
733        let sources: Vec<_> = task
734            .runs
735            .iter()
736            .collect::<std::collections::BTreeSet<_>>()
737            .into_iter()
738            .cloned()
739            .collect();
740        let mut useful = Vec::new();
741        let mut selected = Vec::new();
742        let mut public = Vec::new();
743        let mut unreadable = false;
744        for id in &sources {
745            let Ok(state) = RunState::load_under(id, home) else {
746                unreadable = true;
747                continue;
748            };
749            if state.released() || state.retired_to.is_some() {
750                continue;
751            }
752            if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
753                selected.push(state.clone());
754            }
755            if state.pr.is_some() && state.pr_taken_over_by.is_none() {
756                public.push(state.clone());
757            }
758            if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
759                useful.push(state);
760            }
761        }
762        // Prefer a durable owner-approved selection, then one existing PR
763        // owner, then an unambiguous useful source. Distinct historical changes
764        // without an owner selection cannot be resolved by creation time.
765        let pool = if !selected.is_empty() {
766            selected
767        } else if !public.is_empty() {
768            public
769        } else {
770            useful
771        };
772        if unreadable || pool.len() != 1 {
773            report.retained.push(format!(
774                "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
775                task.id,
776                sources.len()
777            ));
778            continue;
779        }
780        let authority = &pool[0];
781        if authority.candidates.is_empty() {
782            continue;
783        }
784        let latest = &authority.id;
785        for id in &sources {
786            if id == latest || report.retired >= 5 {
787                continue;
788            }
789            if let Ok(state) = RunState::load_under(id, home)
790                && state.retired_to.as_ref() == Some(latest)
791                && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
792            {
793                continue;
794            }
795            let result = async {
796                ensure!(
797                    !task
798                        .resume_override
799                        .as_ref()
800                        .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
801                    "operator-pinned recovery run"
802                );
803                ensure!(
804                    !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
805                    "selected replacement recovery source"
806                );
807                let _owner = crate::pr_head::owner_lock(home, id)?;
808                let mut state = RunState::load_under(id, home)?;
809                ensure!(
810                    state.repo == authority.repo,
811                    "repository differs from recovery authority"
812                );
813                ensure!(
814                    !crate::pr_recovery::explicit_sources_retained(home, id),
815                    "explicitly retained recovery source"
816                );
817                ensure!(
818                    state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
819                        == Liveness::Dead,
820                    "run live or ownership unknown"
821                );
822                let _driver = crate::driver::maintenance(home, id)?;
823                ensure!(
824                    crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
825                    "within grace period"
826                );
827                ensure!(
828                    !open.iter().any(|q| q.run == *id && q.status.open()),
829                    "unanswered owner decision"
830                );
831                ensure!(
832                    state.pr.is_none()
833                        && state.pr_head.is_none()
834                        && state.pr_recovery.is_none()
835                        && state.land_approval.is_none()
836                        && state.merge_decision.is_none()
837                        && state.reviews.is_empty()
838                        && state.recovery.is_none()
839                        && state.candidate_preservation.is_empty()
840                        && state.interrupted_implementations.is_empty()
841                        && !state.released(),
842                    "PR ownership or recovery checkpoint"
843                );
844                let proof = prove_empty(home, &state, latest).await?;
845                commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
846                // Durable retirement precedes removal. A crash retries removal
847                // using the same proof; it cannot make this attempt resumable.
848                for path in &proof.paths {
849                    ensure!(
850                        git::worktree_remove_clean(&state.repo, path).await?,
851                        "Git refused clean worktree removal"
852                    );
853                }
854                Ok::<_, anyhow::Error>(())
855            }
856            .await;
857            match result {
858                Ok(()) => report.retired += 1,
859                Err(e) => report.retained.push(format!("{id}: {e:#}")),
860            }
861        }
862    }
863}
864
865struct EmptyProof {
866    paths: Vec<PathBuf>,
867    _outputs: Vec<OutputLease>,
868    _cargo: Vec<File>,
869}
870
871// Compare actual source bytes with the original tree independently of index
872// stat caches, assume-unchanged flags, sparse checkout and fsmonitor hints.
873async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
874    let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
875    let mut files = Vec::new();
876    let mut blobs = Vec::new();
877    for record in tree.split('\0').filter(|s| !s.is_empty()) {
878        let (header, name) = record.split_once('\t').context("unknown tree record")?;
879        let parts: Vec<_> = header.split_whitespace().collect();
880        ensure!(
881            parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
882            "unsupported tracked source kind"
883        );
884        let path = worktree.join(name);
885        let meta = std::fs::symlink_metadata(&path)?;
886        ensure!(
887            meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
888            "noncanonical tracked source"
889        );
890        #[cfg(unix)]
891        {
892            use std::os::unix::fs::PermissionsExt as _;
893            ensure!(
894                (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
895                "tracked executable mode differs from original base"
896            );
897        }
898        files.push(path.to_str().context("non UTF-8 source")?.to_owned());
899        blobs.push(parts[2].to_owned());
900    }
901    for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
902        let mut args = vec!["hash-object", "--no-filters", "--"];
903        args.extend(paths.iter().map(String::as_str));
904        let actual = git::git(worktree, &args).await?;
905        ensure!(
906            actual.lines().eq(expected.iter().map(String::as_str)),
907            "tracked bytes differ from original base"
908        );
909    }
910    Ok(())
911}
912
913async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
914    let id = &state.id;
915    let paths = worktrees(state).await?;
916    let mut guards = Vec::new();
917    let mut cargo_guards = Vec::new();
918    for path in &paths {
919        guards.push(OutputLease::take(path, true)?);
920        ensure!(
921            git::rev_parse(path, "HEAD").await? == state.base_commit,
922            "unique or unrecorded commits"
923        );
924        ensure!(
925            git::git(
926                path,
927                &[
928                    "--no-optional-locks",
929                    "status",
930                    "--porcelain",
931                    "--untracked-files=all"
932                ]
933            )
934            .await?
935            .is_empty(),
936            "tracked or untracked source changes"
937        );
938        ensure!(
939            git::git(
940                path,
941                &[
942                    "--no-optional-locks",
943                    "status",
944                    "--porcelain",
945                    "--untracked-files=all",
946                    "--ignored",
947                    "--",
948                    ".",
949                    ":(exclude)target"
950                ]
951            )
952            .await?
953            .is_empty(),
954            "source changes, ignored files or build outputs"
955        );
956        tracked_base(path, &state.base_commit).await?;
957        match std::fs::symlink_metadata(path.join("target")) {
958            Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
959            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
960            Err(error) => return Err(error.into()),
961        }
962        for checkpoint in [
963            "rebase-merge",
964            "rebase-apply",
965            "sequencer",
966            "BISECT_LOG",
967            "REVERT_HEAD",
968            "AUTO_MERGE",
969            "MERGE_MSG",
970            "CHERRY_PICK_HEAD",
971            "MERGE_HEAD",
972            "REBASE_HEAD",
973        ] {
974            let location = git::git(
975                path,
976                &[
977                    "rev-parse",
978                    "--path-format=absolute",
979                    "--git-path",
980                    checkpoint,
981                ],
982            )
983            .await?;
984            ensure!(
985                matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
986                "retained Git recovery metadata: {checkpoint}"
987            );
988        }
989        // Git also initializes ORIG_HEAD when creating a worktree.
990        // Only an origin outside the base is a recovery source.
991        let origin_path = git::git(
992            path,
993            &[
994                "rev-parse",
995                "--path-format=absolute",
996                "--git-path",
997                "ORIG_HEAD",
998            ],
999        )
1000        .await?;
1001        match std::fs::symlink_metadata(origin_path.trim()) {
1002            Ok(meta) => {
1003                ensure!(
1004                    meta.is_file() && !meta.file_type().is_symlink(),
1005                    "unknown ORIG_HEAD artifact"
1006                );
1007                git::rev_parse(path, "ORIG_HEAD")
1008                    .await
1009                    .context("unreadable ORIG_HEAD")?;
1010            }
1011            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1012            Err(e) => return Err(e.into()),
1013        }
1014        if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1015            ensure!(
1016                git::git_raw(
1017                    path,
1018                    &["merge-base", "--is-ancestor", &original, &state.base_commit]
1019                )
1020                .await?
1021                .ok(),
1022                "ORIG_HEAD retains commits outside the original base"
1023            );
1024        }
1025        let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1026        for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1027            ensure!(
1028                git::git_raw(
1029                    path,
1030                    &["merge-base", "--is-ancestor", head, &state.base_commit]
1031                )
1032                .await?
1033                .ok(),
1034                "worktree reflog retains commits outside the original base"
1035            );
1036        }
1037        for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1038            ensure!(
1039                !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1040                    .await?
1041                    .ok(),
1042                "unfinished Git operation"
1043            );
1044        }
1045    }
1046    // Missing worktrees cannot establish that unrecorded prep edits
1047    // were empty. Recorded candidates must also point at the base.
1048    if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1049        return Ok(EmptyProof {
1050            paths,
1051            _outputs: guards,
1052            _cargo: cargo_guards,
1053        });
1054    }
1055    ensure!(!paths.is_empty(), "no worktrees to prove empty");
1056    for candidate in &state.candidates {
1057        ensure!(
1058            (paths.contains(&candidate.worktree)
1059                || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1060                && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1061            "candidate recovery source"
1062        );
1063    }
1064    let pattern = format!("refs/heads/magi/{}/", state.short());
1065    let branches = git::git(
1066        &state.repo,
1067        &["for-each-ref", "--format=%(objectname)", &pattern],
1068    )
1069    .await?;
1070    ensure!(
1071        branches.lines().all(|head| head == state.base_commit),
1072        "unrecorded candidate branch changes"
1073    );
1074    Ok(EmptyProof {
1075        paths,
1076        _outputs: guards,
1077        _cargo: cargo_guards,
1078    })
1079}
1080
1081/// The only task-write critical section in retirement. All Git proof and
1082/// deletion happens outside it, under the run's owner/driver/output leases.
1083/// Atomic task writes make the earlier snapshot consistent; this final compare
1084/// rejects every intervening owner selection, hold, edit or history update.
1085fn commit_retirement(
1086    home: &Path,
1087    queue: &crate::queue::Queue,
1088    expected: &crate::queue::Task,
1089    state: &mut RunState,
1090    authority: &str,
1091    sources: &[String],
1092    now: Timestamp,
1093) -> Result<()> {
1094    let _write = queue.lock_task(&expected.id)?;
1095    let current = queue.get(&expected.id)?;
1096    ensure!(
1097        serde_json::to_value(&current)? == serde_json::to_value(expected)?,
1098        "task changed during retirement proof; retaining worktrees for the current owner decision"
1099    );
1100    ensure!(
1101        !current
1102            .resume_override
1103            .as_ref()
1104            .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1105            && !current
1106                .replacement
1107                .as_ref()
1108                .is_some_and(|pin| pin.run == state.id),
1109        "current owner selected this recovery source"
1110    );
1111    ensure!(
1112        !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1113        "explicitly retained recovery source"
1114    );
1115    ensure!(
1116        !crate::ask::Questions::at(home.join("questions"))
1117            .list()
1118            .iter()
1119            .any(|q| q.status.open()
1120                && (q.run == state.id
1121                    || q.run == current.id
1122                    || current.blocked_by.contains(&q.id))),
1123        "owner decision arrived during retirement proof"
1124    );
1125    let dir = home.join("recovery-lineages");
1126    std::fs::create_dir_all(&dir)?;
1127    let tmp = dir.join(format!("{}.tmp", current.id));
1128    let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1129        "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1130    std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1131    std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1132    state.retired_to = Some(authority.to_owned());
1133    state.event(
1134        "retention",
1135        format!(
1136            "empty attempt retired to {authority}; recovery sources: {}",
1137            sources.join(", ")
1138        ),
1139    );
1140    state.save_under(home)?;
1141    // No Git command or worktree removal may be added before this guard drops.
1142    Ok(())
1143}
1144
1145/// Wait for the emergency reserve to be exhausted. The normal floor requests
1146/// a checkpoint first; half the floor stops owned writers before node completion.
1147/// A failed measurement also closes admission rather than allowing blind writes.
1148pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1149    if min_free == 0 {
1150        return std::future::pending().await;
1151    }
1152    loop {
1153        for path in &paths {
1154            let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1155            match crate::disk::free_bytes(existing) {
1156                Ok(free) if free >= min_free / 2 => {}
1157                Ok(free) => {
1158                    return format!(
1159                        "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1160                    );
1161                }
1162                Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1163            }
1164        }
1165        tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1166    }
1167}
1168
1169#[cfg(all(test, unix))]
1170mod tests {
1171    use super::*;
1172    use crate::{
1173        config::{Config, Disk},
1174        queue::{Source, Task},
1175        run::{Candidate, RunStatus},
1176    };
1177
1178    struct Fx {
1179        _temp: tempfile::TempDir,
1180        home: PathBuf,
1181        repo: PathBuf,
1182        bay: PathBuf,
1183        base: String,
1184    }
1185    impl Fx {
1186        async fn new() -> Self {
1187            let temp = tempfile::tempdir().unwrap();
1188            let root = temp.path().canonicalize().unwrap();
1189            let home = root.join("home");
1190            let repo = root.join("repo");
1191            let bay = root.join("worktrees");
1192            std::fs::create_dir_all(&repo).unwrap();
1193            std::fs::create_dir_all(home.join("runs")).unwrap();
1194            git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1195            git::git(&repo, &["config", "user.name", "Fixture"])
1196                .await
1197                .unwrap();
1198            git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1199                .await
1200                .unwrap();
1201            git::git(&repo, &["config", "commit.gpgsign", "false"])
1202                .await
1203                .unwrap();
1204            std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1205            std::fs::write(repo.join("source"), "base\n").unwrap();
1206            git::git(&repo, &["add", "."]).await.unwrap();
1207            git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1208            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1209            Self {
1210                _temp: temp,
1211                home,
1212                repo,
1213                bay,
1214                base,
1215            }
1216        }
1217        async fn run(&self, tag: &str, useful: bool) -> RunState {
1218            let mut cfg = Config::default();
1219            cfg.graph.worktree_root = Some(self.bay.clone());
1220            let mut state = RunState::new(
1221                self.repo.clone(),
1222                "main".into(),
1223                self.base.clone(),
1224                "task".into(),
1225                cfg,
1226            );
1227            state.id = format!("20260101-000000-{tag}");
1228            state.status = RunStatus::Blocked;
1229            state.driver_exited = true;
1230            let wt = state.worktree_root().join("cand-A");
1231            let branch = format!("magi/{tag}/A");
1232            git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1233                .await
1234                .unwrap();
1235            if useful {
1236                std::fs::write(wt.join("source"), tag).unwrap();
1237                git::git(&wt, &["add", "."]).await.unwrap();
1238                git::git(&wt, &["commit", "-m", "candidate change"])
1239                    .await
1240                    .unwrap();
1241            }
1242            state.candidates.push(Candidate {
1243                index: 0,
1244                label: 'A',
1245                agent: "fixture".into(),
1246                branch,
1247                worktree: wt,
1248                summary: String::new(),
1249                stat: String::new(),
1250                files: usize::from(useful),
1251                commits: usize::from(useful),
1252                empty: !useful,
1253                failed: None,
1254                verified_noop: None,
1255                duration_ms: 0,
1256                folded: false,
1257            });
1258            state.save_under(&self.home).unwrap();
1259            state
1260        }
1261        fn disk() -> Disk {
1262            Disk {
1263                worktree_limit_bytes: 1,
1264                worktree_reclaim_bytes: 1024 * 1024,
1265                fold_grace_secs: 0,
1266                min_free_bytes: 0,
1267                ..Disk::default()
1268            }
1269        }
1270        fn target(wt: &Path) {
1271            let target = wt.join("target");
1272            std::fs::create_dir_all(target.join("debug")).unwrap();
1273            let mut artifact = [1; 200];
1274            artifact[..4].copy_from_slice(b"\x7fELF");
1275            std::fs::write(target.join("debug/output"), artifact).unwrap();
1276            let output = target
1277                .join("debug/output")
1278                .display()
1279                .to_string()
1280                .replace('\\', "\\\\")
1281                .replace(' ', "\\ ");
1282            std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1283            std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1284            std::fs::write(
1285                target.join("CACHEDIR.TAG"),
1286                "Signature: 8a477f597d28d172789f06886806bc55d\n",
1287            )
1288            .unwrap();
1289            std::fs::write(
1290                target.join(".rustc_info.json"),
1291                "{\"rustc_fingerprint\":123}",
1292            )
1293            .unwrap();
1294        }
1295        fn now() -> Timestamp {
1296            Timestamp::now() + jiff::SignedDuration::from_secs(2)
1297        }
1298        fn task(&self, runs: &[RunState]) -> Task {
1299            let mut task = Task::new(
1300                "fixture".into(),
1301                "task".into(),
1302                self.repo.clone(),
1303                Source::Human,
1304            );
1305            task.runs = runs.iter().map(|s| s.id.clone()).collect();
1306            crate::queue::Queue::at(self.home.join("queue"))
1307                .put(&mut task)
1308                .unwrap();
1309            task
1310        }
1311    }
1312
1313    #[tokio::test]
1314    async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
1315        for remove_root in [true, false] {
1316            let f = Fx::new().await;
1317            let stale = Fx::new().await;
1318            let mut missing = stale.run("aa00", false).await;
1319            Fx::target(&missing.candidates[0].worktree);
1320            missing.save_under(&f.home).unwrap();
1321            if remove_root {
1322                std::fs::remove_dir_all(missing.worktree_root()).unwrap();
1323                // An existing non-repository proves the absent root is checked
1324                // before querying Git, even when no registrations are cached.
1325                std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
1326            } else {
1327                std::fs::remove_dir_all(&stale.repo).unwrap();
1328            }
1329            let healthy = f.run("aa01", false).await;
1330            Fx::target(&healthy.candidates[0].worktree);
1331
1332            let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1333            assert!(report.failures.is_empty(), "{report:?}");
1334            assert_eq!(report.freed_bytes, 200, "{report:?}");
1335            assert!(
1336                !healthy.candidates[0]
1337                    .worktree
1338                    .join("target/debug/output")
1339                    .exists()
1340            );
1341            if !remove_root {
1342                // Without the repository, ownership cannot be validated.
1343                assert!(
1344                    missing.candidates[0]
1345                        .worktree
1346                        .join("target/debug/output")
1347                        .exists()
1348                );
1349            }
1350        }
1351    }
1352
1353    #[tokio::test]
1354    async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
1355        let f = Fx::new().await;
1356        let mut missing = f.run("aa00", false).await;
1357        Fx::target(&missing.candidates[0].worktree);
1358        missing.repo = missing.candidates[0].worktree.clone();
1359        missing.save_under(&f.home).unwrap();
1360        std::fs::remove_dir_all(&missing.repo).unwrap();
1361        assert!(missing.worktree_root().is_dir());
1362        assert!(f.repo.is_dir());
1363
1364        let healthy = f.run("aa01", false).await;
1365        Fx::target(&healthy.candidates[0].worktree);
1366
1367        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1368        assert!(report.failures.is_empty(), "{report:?}");
1369        assert_eq!(report.freed_bytes, 200, "{report:?}");
1370        assert!(
1371            !healthy.candidates[0]
1372                .worktree
1373                .join("target/debug/output")
1374                .exists()
1375        );
1376    }
1377
1378    #[tokio::test]
1379    async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
1380        let f = Fx::new().await;
1381        let broken = Fx::new().await;
1382        let mut invalid = broken.run("aa00", false).await;
1383        Fx::target(&invalid.candidates[0].worktree);
1384        invalid.save_under(&f.home).unwrap();
1385        std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
1386        let healthy = f.run("aa01", false).await;
1387        Fx::target(&healthy.candidates[0].worktree);
1388
1389        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1390        assert_eq!(report.failures.len(), 1, "{report:?}");
1391        assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
1392        assert_eq!(report.freed_bytes, 200, "{report:?}");
1393        assert!(
1394            !healthy.candidates[0]
1395                .worktree
1396                .join("target/debug/output")
1397                .exists()
1398        );
1399        assert!(
1400            invalid.candidates[0]
1401                .worktree
1402                .join("target/debug/output")
1403                .exists()
1404        );
1405    }
1406
1407    async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
1408        let path = reconcile::run(&f.home, &task.id, &source.id, None)
1409            .await
1410            .unwrap();
1411        serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
1412    }
1413
1414    #[tokio::test]
1415    async fn reconciliation_standing_chat_requires_exact_current_owner_authority() {
1416        use crate::{
1417            chat_control::{Operation, dispatch},
1418            talk::{self, Talks},
1419        };
1420        let f = Fx::new().await;
1421        let source = f.run("ae00", true).await;
1422        let clean = f.run("ae01", false).await;
1423        let queue = crate::queue::Queue::at(f.home.join("queue"));
1424        let talks = Talks::at(f.home.join("talks"));
1425        let mut conversation: talk::Talk = serde_json::from_value(serde_json::json!({
1426            "schema":1,"id":"talk-owner","repo":f.repo,"agent":"worker","status":"open","turns":[],
1427            "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
1428            "seat":crate::agent::SeatState::new("talk","worker",1)
1429        }))
1430        .unwrap();
1431        talks.put(&mut conversation).unwrap();
1432        let mut task = f.task(&[source.clone(), clean.clone()]);
1433        let operation = |apply: Option<&str>, quote: Option<&str>| Operation::ReconcileRetention {
1434            task: task.id.clone(),
1435            authority: source.id.clone(),
1436            apply: apply.map(str::to_owned),
1437            quote: quote.map(str::to_owned),
1438        };
1439        assert!(
1440            dispatch(
1441                &queue,
1442                &f.home,
1443                &talks,
1444                &conversation.id,
1445                operation(None, None)
1446            )
1447            .await
1448            .is_err()
1449        );
1450        task.chat = Some(conversation.id.clone());
1451        queue.put(&mut task).unwrap();
1452        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1453        let operation = |apply: Option<&str>, quote: Option<&str>| Operation::ReconcileRetention {
1454            task: task.id.clone(),
1455            authority: source.id.clone(),
1456            apply: apply.map(str::to_owned),
1457            quote: quote.map(str::to_owned),
1458        };
1459        let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
1460            .await
1461            .unwrap();
1462        let legacy: serde_json::Value =
1463            serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
1464        let legacy_id = legacy["id"].as_str().unwrap();
1465        let legacy_words = reconcile::apply_instruction(legacy_id, &task.id, &source.id);
1466        talk::record(&mut conversation, &talks, &legacy_words, vec![]).unwrap();
1467        assert!(
1468            dispatch(
1469                &queue,
1470                &f.home,
1471                &talks,
1472                &conversation.id,
1473                operation(Some(legacy_id), Some(&legacy_words))
1474            )
1475            .await
1476            .is_err()
1477        );
1478        let summary = dispatch(
1479            &queue,
1480            &f.home,
1481            &talks,
1482            &conversation.id,
1483            operation(None, None),
1484        )
1485        .await
1486        .unwrap();
1487        assert!(summary.contains("Complete review summary"));
1488        assert!(summary.contains(&source.id) && summary.contains(&clean.id));
1489        let root = f.home.join("retention-reconciliations");
1490        let displayed: serde_json::Value =
1491            serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
1492        let id = displayed["plan"].as_str().unwrap();
1493        let dir = root.join(id);
1494        let plan: serde_json::Value =
1495            serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
1496        assert_eq!(plan["version"], 2);
1497        assert!(clean.candidates[0].worktree.exists());
1498        assert!(!dir.join("approved.json").exists());
1499        let expected = reconcile::apply_instruction(id, &task.id, &source.id);
1500        for words in [
1501            "continue".to_owned(),
1502            format!("{expected}\nand another plan"),
1503            reconcile::apply_instruction("wrong-plan", &task.id, &source.id),
1504            reconcile::apply_instruction(id, "wrong-task", &source.id),
1505            reconcile::apply_instruction(id, &task.id, "wrong-authority"),
1506        ] {
1507            talk::record(&mut conversation, &talks, &words, vec![]).unwrap();
1508            assert!(
1509                dispatch(
1510                    &queue,
1511                    &f.home,
1512                    &talks,
1513                    &conversation.id,
1514                    operation(Some(id), Some(&words))
1515                )
1516                .await
1517                .is_err()
1518            );
1519            assert!(!dir.join("approved.json").exists());
1520        }
1521        talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1522        assert!(
1523            dispatch(
1524                &queue,
1525                &f.home,
1526                &talks,
1527                &conversation.id,
1528                operation(Some(id), None)
1529            )
1530            .await
1531            .is_err()
1532        );
1533        conversation.pending = "wait".into();
1534        talks.put(&mut conversation).unwrap();
1535        assert!(
1536            dispatch(
1537                &queue,
1538                &f.home,
1539                &talks,
1540                &conversation.id,
1541                operation(Some(id), Some(&expected))
1542            )
1543            .await
1544            .is_err()
1545        );
1546        conversation.pending.clear();
1547        talks.put(&mut conversation).unwrap();
1548        talk::record(&mut conversation, &talks, "wait", vec![]).unwrap();
1549        let error = reconcile::interrupt_chat_after_fence(
1550            &f.home,
1551            &task,
1552            id,
1553            reconcile::ChatRequest {
1554                talks: &talks,
1555                conversation: &conversation.id,
1556                quote: Some(&expected),
1557            },
1558        )
1559        .await
1560        .unwrap_err();
1561        assert!(
1562            !error.to_string().contains("simulated interruption"),
1563            "{error:#}"
1564        );
1565        assert!(!dir.join("approved.json").exists());
1566        assert!(
1567            RunState::load_under(&clean.id, &f.home)
1568                .unwrap()
1569                .retired_to
1570                .is_none()
1571        );
1572        talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1573        let error = reconcile::interrupt_chat_after_fence(
1574            &f.home,
1575            &task,
1576            id,
1577            reconcile::ChatRequest {
1578                talks: &talks,
1579                conversation: &conversation.id,
1580                quote: Some(&expected),
1581            },
1582        )
1583        .await
1584        .unwrap_err();
1585        assert!(
1586            error.to_string().contains("simulated interruption"),
1587            "{error:#}"
1588        );
1589        assert!(clean.candidates[0].worktree.exists());
1590        assert!(dir.join("approved.json").exists());
1591        talk::record(&mut conversation, &talks, "wait", vec![]).unwrap();
1592        assert!(
1593            dispatch(
1594                &queue,
1595                &f.home,
1596                &talks,
1597                &conversation.id,
1598                operation(Some(id), Some(&expected))
1599            )
1600            .await
1601            .is_err()
1602        );
1603        talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1604        dispatch(
1605            &queue,
1606            &f.home,
1607            &talks,
1608            &conversation.id,
1609            operation(Some(id), Some(&expected)),
1610        )
1611        .await
1612        .unwrap();
1613        assert!(!clean.candidates[0].worktree.exists());
1614        assert!(source.candidates[0].worktree.exists());
1615        let audit: serde_json::Value =
1616            serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
1617        assert_eq!(audit["version"], 2);
1618        assert_eq!(audit["owner"]["conversation"], conversation.id);
1619        assert_eq!(audit["owner"]["evidence"]["instruction"], expected);
1620        talk::record(&mut conversation, &talks, "continue", vec![]).unwrap();
1621        assert!(
1622            dispatch(
1623                &queue,
1624                &f.home,
1625                &talks,
1626                &conversation.id,
1627                operation(Some(id), Some(&expected))
1628            )
1629            .await
1630            .is_err()
1631        );
1632        talk::record(&mut conversation, &talks, &expected, vec![]).unwrap();
1633        dispatch(
1634            &queue,
1635            &f.home,
1636            &talks,
1637            &conversation.id,
1638            operation(Some(id), Some(&expected)),
1639        )
1640        .await
1641        .unwrap();
1642    }
1643
1644    #[tokio::test]
1645    async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
1646        let f = Fx::new().await;
1647        let source = f.run("af00", true).await;
1648        let mut clean = f.run("af01", false).await;
1649        let mut dirty = f.run("af02", false).await;
1650        for state in [&mut clean, &mut dirty] {
1651            state.status = RunStatus::Prep;
1652            state.prep_complete = Some(false);
1653            state.candidates[0].empty = false;
1654            state.save_under(&f.home).unwrap();
1655        }
1656        std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
1657        let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
1658        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1659        let plan = reconciliation_plan(&f, &task, &source).await;
1660        let selected: Vec<_> = plan["candidates"]
1661            .as_array()
1662            .unwrap()
1663            .iter()
1664            .filter(|e| e["selected"] == true)
1665            .collect();
1666        assert_eq!(selected.len(), 1, "{plan:#}");
1667        assert_eq!(selected[0]["run"], clean.id);
1668        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1669            .await
1670            .unwrap();
1671        assert!(!clean.candidates[0].worktree.exists());
1672        assert!(dirty.candidates[0].worktree.exists());
1673        assert!(
1674            RunState::load_under(&dirty.id, &f.home)
1675                .unwrap()
1676                .retired_to
1677                .is_none()
1678        );
1679    }
1680
1681    #[tokio::test]
1682    async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
1683        let f = Fx::new().await;
1684        let mut source = f.run("aa00", true).await;
1685        let alternative = f.run("aa01", true).await;
1686        let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
1687            .await
1688            .unwrap();
1689        source.reviewed_commits = Some(vec![head.clone()]);
1690        source.save_under(&f.home).unwrap();
1691        let mut runs = vec![source.clone(), alternative.clone()];
1692        for i in 2..14 {
1693            runs.push(f.run(&format!("aa{i:02}"), false).await);
1694        }
1695        let mut task = f.task(&runs);
1696        task.pr_recovery = Some(crate::pr_recovery::Selection {
1697            url: "https://example.invalid/pull/1".into(),
1698            branch: "selected".into(),
1699            head: head.clone(),
1700            source_head: Some(head.clone()),
1701            source_run: Some(source.id.clone()),
1702            source_proof: None,
1703            disposition: Some(crate::pr_recovery::Disposition::Land),
1704            instruction: task.instruction.clone(),
1705            approved_at: Timestamp::now(),
1706        });
1707        source.pr_recovery = task.pr_recovery.clone();
1708        source.save_under(&f.home).unwrap();
1709        runs[0] = source.clone();
1710        crate::queue::Queue::at(f.home.join("queue"))
1711            .put(&mut task)
1712            .unwrap();
1713        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1714        let task_before = serde_json::to_value(&task).unwrap();
1715        let mut report = Report::default();
1716        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
1717        assert_eq!(
1718            report.retired, 0,
1719            "automatic cleanup must still refuse blanket markers"
1720        );
1721        assert!(
1722            report
1723                .retained
1724                .iter()
1725                .any(|reason| reason.contains("explicitly retained"))
1726        );
1727        let mut total = 0;
1728        for expected in [5, 5, 2, 0] {
1729            let plan = reconciliation_plan(&f, &task, &source).await;
1730            let selected: Vec<_> = plan["candidates"]
1731                .as_array()
1732                .unwrap()
1733                .iter()
1734                .filter(|e| e["selected"] == true)
1735                .collect();
1736            assert_eq!(selected.len(), expected, "{plan:#}");
1737            for e in &selected {
1738                assert!(
1739                    RunState::load_under(e["run"].as_str().unwrap(), &f.home)
1740                        .unwrap()
1741                        .retired_to
1742                        .is_none()
1743                );
1744            }
1745            let operation = plan["id"].as_str().unwrap();
1746            for _ in 0..2 {
1747                reconcile::run(&f.home, &task.id, &source.id, Some(operation))
1748                    .await
1749                    .unwrap();
1750            }
1751            total += expected;
1752        }
1753        assert_eq!(total, 12);
1754        for run in &runs {
1755            let dir = f.home.join("runs").join(&run.id);
1756            assert!(dir.join("run.json").exists());
1757            assert!(dir.join("retain-recovery-sources").exists());
1758            assert!(
1759                git::branch_exists(&f.repo, &run.candidates[0].branch)
1760                    .await
1761                    .unwrap()
1762            );
1763        }
1764        assert!(source.candidates[0].worktree.exists());
1765        assert!(alternative.candidates[0].worktree.exists());
1766        assert_eq!(
1767            git::rev_parse(&f.repo, &source.candidates[0].branch)
1768                .await
1769                .unwrap(),
1770            head
1771        );
1772        assert_eq!(
1773            serde_json::to_value(
1774                crate::queue::Queue::at(f.home.join("queue"))
1775                    .get(&task.id)
1776                    .unwrap()
1777            )
1778            .unwrap(),
1779            task_before
1780        );
1781        assert_eq!(
1782            runs.iter()
1783                .filter(|r| RunState::load_under(&r.id, &f.home)
1784                    .unwrap()
1785                    .retired_to
1786                    .is_none())
1787                .count(),
1788            2
1789        );
1790    }
1791
1792    #[tokio::test]
1793    async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
1794        let f = Fx::new().await;
1795        let source = f.run("ab00", true).await;
1796        let mut runs = vec![source.clone()];
1797        for i in 1..9 {
1798            runs.push(f.run(&format!("ab{i:02}"), false).await);
1799        }
1800        std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
1801        std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
1802        std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
1803        std::fs::write(
1804            runs[3].candidates[0].worktree.join("scratch/source"),
1805            "ignored source",
1806        )
1807        .unwrap();
1808        std::fs::write(
1809            runs[4].worktree_root().join("unknown"),
1810            "unregistered source",
1811        )
1812        .unwrap();
1813        let checkpoint = git::git(
1814            &runs[5].candidates[0].worktree,
1815            &[
1816                "rev-parse",
1817                "--path-format=absolute",
1818                "--git-path",
1819                "sequencer",
1820            ],
1821        )
1822        .await
1823        .unwrap();
1824        std::fs::create_dir(checkpoint.trim()).unwrap();
1825        runs[6].driver_exited = false;
1826        runs[6].driver_pid = Some(std::process::id());
1827        runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
1828        runs[6].save_under(&f.home).unwrap();
1829        runs[7].driver_exited = false;
1830        runs[7].driver_pid = None;
1831        runs[7].driver_claimed = None; // Legacy record without first-claim evidence.
1832        runs[7].save_under(&f.home).unwrap();
1833        assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
1834
1835        let hidden = &runs[8].candidates[0].worktree;
1836        git::git(hidden, &["update-index", "--assume-unchanged", "source"])
1837            .await
1838            .unwrap();
1839        std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
1840        assert!(
1841            git::git(hidden, &["status", "--porcelain"])
1842                .await
1843                .unwrap()
1844                .is_empty()
1845        );
1846        let task = f.task(&runs);
1847        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1848        let plan = reconciliation_plan(&f, &task, &source).await;
1849        assert!(
1850            plan["candidates"]
1851                .as_array()
1852                .unwrap()
1853                .iter()
1854                .all(|e| e["selected"] == false),
1855            "{plan:#}"
1856        );
1857        for run in &runs {
1858            assert!(run.candidates[0].worktree.exists());
1859        }
1860    }
1861
1862    #[tokio::test]
1863    async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
1864        let f = Fx::new().await;
1865        let source = f.run("ac00", true).await;
1866        let empty = f.run("ac01", false).await;
1867        let task = f.task(&[source.clone(), empty.clone()]);
1868        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1869        let plan = reconciliation_plan(&f, &task, &source).await;
1870        std::fs::write(
1871            f.home.join("runs").join(&source.id).join("new-report"),
1872            "new review checkpoint",
1873        )
1874        .unwrap();
1875        assert!(
1876            reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
1877                .await
1878                .is_err()
1879        );
1880        assert!(empty.candidates[0].worktree.exists());
1881        assert!(
1882            RunState::load_under(&empty.id, &f.home)
1883                .unwrap()
1884                .retired_to
1885                .is_none()
1886        );
1887        let marker = f
1888            .home
1889            .join("runs")
1890            .join(&empty.id)
1891            .join("retain-recovery-sources");
1892        std::fs::write(&marker, "different owner").unwrap();
1893        let plan = reconciliation_plan(&f, &task, &source).await;
1894        assert!(
1895            plan["candidates"]
1896                .as_array()
1897                .unwrap()
1898                .iter()
1899                .all(|e| e["selected"] == false)
1900        );
1901    }
1902
1903    #[tokio::test]
1904    async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
1905        for partial in [false, true] {
1906            let f = Fx::new().await;
1907            let source = f.run("ab10", true).await;
1908            let first = f.run("ab11", false).await;
1909            let second = f.run("ab12", false).await;
1910            let task = f.task(&[source.clone(), first.clone(), second.clone()]);
1911            crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1912            let plan = reconciliation_plan(&f, &task, &source).await;
1913            let id = plan["id"].as_str().unwrap();
1914            let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
1915            // Matching bytes without this operation's fence are not trusted.
1916            std::fs::write(&staging, b"").unwrap();
1917            assert!(
1918                reconcile::run(&f.home, &task.id, &source.id, Some(id))
1919                    .await
1920                    .is_err()
1921            );
1922            assert!(staging.exists());
1923            std::fs::remove_file(&staging).unwrap();
1924            let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
1925                .await
1926                .unwrap_err();
1927            assert!(
1928                error.to_string().contains("before run state rename"),
1929                "{error:#}"
1930            );
1931            assert!(
1932                RunState::load_under(&first.id, &f.home)
1933                    .unwrap()
1934                    .retired_to
1935                    .is_some()
1936            );
1937            assert!(
1938                RunState::load_under(&second.id, &f.home)
1939                    .unwrap()
1940                    .retired_to
1941                    .is_none()
1942            );
1943            assert!(first.candidates[0].worktree.exists());
1944            assert!(second.candidates[0].worktree.exists());
1945            let interrupted = std::fs::read(&staging).unwrap();
1946            // An unrelated file must survive a refused retry unchanged.
1947            std::fs::write(&staging, b"unrelated recovery source").unwrap();
1948            assert!(
1949                reconcile::run(&f.home, &task.id, &source.id, Some(id))
1950                    .await
1951                    .is_err()
1952            );
1953            assert_eq!(
1954                std::fs::read(&staging).unwrap(),
1955                b"unrelated recovery source"
1956            );
1957            std::fs::write(&staging, interrupted).unwrap();
1958            reconcile::run(&f.home, &task.id, &source.id, Some(id))
1959                .await
1960                .unwrap();
1961            assert!(!staging.exists());
1962            assert!(!first.candidates[0].worktree.exists());
1963            assert!(!second.candidates[0].worktree.exists());
1964            assert_eq!(
1965                RunState::load_under(&second.id, &f.home)
1966                    .unwrap()
1967                    .retired_to,
1968                Some(source.id)
1969            );
1970        }
1971    }
1972
1973    #[tokio::test]
1974    async fn reconciliation_resumes_only_its_exact_durable_fence() {
1975        let f = Fx::new().await;
1976        let source = f.run("ad00", true).await;
1977        let empty = f.run("ad01", false).await;
1978        let task = f.task(&[source.clone(), empty.clone()]);
1979        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
1980        let plan = reconciliation_plan(&f, &task, &source).await;
1981        let id = plan["id"].as_str().unwrap();
1982        assert!(
1983            reconcile::interrupt_after_fence(&f.home, &task, id)
1984                .await
1985                .is_err()
1986        );
1987        assert!(empty.candidates[0].worktree.exists());
1988        assert_eq!(
1989            RunState::load_under(&empty.id, &f.home)
1990                .unwrap()
1991                .retired_to
1992                .as_ref(),
1993            Some(&source.id)
1994        );
1995        reconcile::run(&f.home, &task.id, &source.id, Some(id))
1996            .await
1997            .unwrap();
1998        assert!(!empty.candidates[0].worktree.exists());
1999        // Model an interrupted removal: restore precisely the approved base worktree.
2000        git::worktree_add_branch(
2001            &f.repo,
2002            &empty.candidates[0].worktree,
2003            "restore-empty",
2004            &f.base,
2005        )
2006        .await
2007        .unwrap();
2008        // Recreated Git administration differs from approval and is refused.
2009        assert!(
2010            reconcile::run(&f.home, &task.id, &source.id, Some(id))
2011                .await
2012                .is_err()
2013        );
2014        assert!(empty.candidates[0].worktree.exists());
2015        let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
2016        changed.prep_failure = Some("new owner edit".into());
2017        changed.save_under(&f.home).unwrap();
2018        assert!(
2019            reconcile::run(&f.home, &task.id, &source.id, Some(id))
2020                .await
2021                .is_err()
2022        );
2023    }
2024
2025    #[tokio::test]
2026    async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
2027        let f = Fx::new().await;
2028        let source = f.run("ae00", true).await;
2029        let mut prep = f.run("ae01", false).await;
2030        let candidate = prep.candidates.pop().unwrap();
2031        prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
2032        prep.save_under(&f.home).unwrap();
2033        let reset = f.run("ae02", false).await;
2034        let wt = &reset.candidates[0].worktree;
2035        std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
2036        git::git(wt, &["add", "."]).await.unwrap();
2037        git::git(wt, &["commit", "-m", "unique implementation"])
2038            .await
2039            .unwrap();
2040        git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
2041        let unknown = f.run("ae03", false).await;
2042        let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
2043        let mut raw: serde_json::Value =
2044            serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
2045        raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
2046        std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
2047        let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
2048        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2049        let plan = reconciliation_plan(&f, &task, &source).await;
2050        let selected: Vec<_> = plan["candidates"]
2051            .as_array()
2052            .unwrap()
2053            .iter()
2054            .filter(|e| e["selected"] == true)
2055            .collect();
2056        assert_eq!(selected.len(), 1, "{plan:#}");
2057        assert_eq!(selected[0]["run"], prep.id);
2058        reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
2059            .await
2060            .unwrap();
2061        assert!(!candidate.worktree.exists());
2062        assert!(reset.candidates[0].worktree.exists());
2063        assert!(unknown.candidates[0].worktree.exists());
2064    }
2065
2066    #[tokio::test]
2067    async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
2068        let f = Fx::new().await;
2069        let mut state = f.run("aaaa", true).await;
2070        let candidate = state.candidates[0].worktree.clone();
2071        std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
2072        Fx::target(&candidate);
2073        for seat in ["review-1", "review-2"] {
2074            let wt = state.worktree_root().join(seat);
2075            git::worktree_add_detached(&f.repo, &wt, &f.base)
2076                .await
2077                .unwrap();
2078            Fx::target(&wt);
2079        }
2080        state.status = RunStatus::Landing;
2081        state.save_under(&f.home).unwrap();
2082        let second = Fx::new().await;
2083        let other = second.run("bbbb", true).await;
2084        // A global store may contain repositories with different worktree roots.
2085        let mut imported = other.clone();
2086        imported.save_under(&f.home).unwrap();
2087        Fx::target(&other.candidates[0].worktree);
2088        let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
2089        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2090        assert!(
2091            candidate.join("target/debug/output").exists(),
2092            "active writer protected"
2093        );
2094        assert!(
2095            !state
2096                .worktree_root()
2097                .join("review-1/target/debug/output")
2098                .exists()
2099        );
2100        assert!(
2101            !state
2102                .worktree_root()
2103                .join("review-2/target/debug/output")
2104                .exists()
2105        );
2106        assert!(
2107            !other.candidates[0]
2108                .worktree
2109                .join("target/debug/output")
2110                .exists()
2111        );
2112        assert!(out.freed_bytes >= 600);
2113        drop(guard);
2114        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2115        assert!(out.freed_bytes >= 200);
2116        assert_eq!(
2117            std::fs::read_to_string(candidate.join("source")).unwrap(),
2118            "uncommitted work"
2119        );
2120        assert!(
2121            git::branch_exists(&f.repo, &state.candidates[0].branch)
2122                .await
2123                .unwrap()
2124        );
2125    }
2126
2127    #[tokio::test]
2128    async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
2129        let f = Fx::new().await;
2130        let state = f.run("cc01", false).await;
2131        let wt = &state.candidates[0].worktree;
2132        Fx::target(wt);
2133        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2134            let dir = wt.join("target").join(dir);
2135            std::fs::create_dir_all(&dir).unwrap();
2136            std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
2137        }
2138        // A dep-info dependency is an input, never deletion authority.
2139        let manifest = wt.join("target/debug/output.d");
2140        let output = wt.join("target/debug/output");
2141        std::fs::write(
2142            &manifest,
2143            format!(
2144                "{}: {}\n",
2145                output.display(),
2146                wt.join("target/debug/notes.txt").display()
2147            ),
2148        )
2149        .unwrap();
2150        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2151        assert_eq!(report.freed_bytes, 200);
2152        assert!(!output.exists());
2153        for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
2154            assert_eq!(
2155                std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
2156                "user evidence"
2157            );
2158        }
2159        // Metadata remains available for inspection and future safe rebuilds.
2160        assert!(manifest.exists());
2161        assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
2162        std::fs::write(&output, "user note replacing an old binary").unwrap();
2163        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2164        assert_eq!(report.freed_bytes, 0);
2165        assert_eq!(
2166            std::fs::read_to_string(&output).unwrap(),
2167            "user note replacing an old binary"
2168        );
2169    }
2170
2171    #[tokio::test]
2172    async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
2173        let f = Fx::new().await;
2174        let state = f.run("cc02", false).await;
2175        let wt = &state.candidates[0].worktree;
2176        Fx::target(wt);
2177        let mut disk = Fx::disk();
2178        disk.worktree_limit_bytes = u64::MAX;
2179        let now = Fx::now();
2180        let first = sweep(&f.home, &disk, now).await;
2181        std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
2182        assert_eq!(
2183            sweep(&f.home, &disk, now).await.measured_bytes,
2184            first.measured_bytes
2185        );
2186        assert!(
2187            sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
2188                .await
2189                .measured_bytes
2190                > first.measured_bytes
2191        );
2192        disk.worktree_limit_bytes = 0;
2193        assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
2194    }
2195
2196    #[tokio::test]
2197    async fn unfinished_source_fold_reports_its_retention_reason() {
2198        let f = Fx::new().await;
2199        let mut source = f.run("cc03", true).await;
2200        let task = f.task(&[source.clone()]);
2201        let error = crate::graph::fold_run(&mut source, true, &f.home)
2202            .await
2203            .unwrap_err();
2204        assert!(error.to_string().contains(&task.id));
2205        assert!(source.candidates[0].worktree.exists());
2206        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2207        let error = crate::graph::fold_run(&mut source, true, &f.home)
2208            .await
2209            .unwrap_err();
2210        assert!(error.to_string().contains("explicitly retained"));
2211    }
2212
2213    #[tokio::test]
2214    async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
2215        let f = Fx::new().await;
2216        let mut source = f.run("cc04", true).await;
2217        let mut task = f.task(&[source.clone()]);
2218        task.hold_manual(Some("queue settlement pending".into()));
2219        let queue = crate::queue::Queue::at(f.home.join("queue"));
2220        queue.put(&mut task).unwrap();
2221        source.merge = Some(crate::run::MergeOutcome {
2222            mode: crate::config::MergeMode::Pr,
2223            ok: true,
2224            detail: "confirmed merged".into(),
2225            empty: false,
2226        });
2227        // Publication success and terminal labels alone are insufficient.
2228        for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
2229            source.status = status;
2230            assert!(
2231                crate::graph::fold_run(&mut source, true, &f.home)
2232                    .await
2233                    .is_err()
2234            );
2235            assert!(source.candidates[0].worktree.exists());
2236        }
2237        git::git(
2238            &f.repo,
2239            &["merge", "--ff-only", &source.candidates[0].branch],
2240        )
2241        .await
2242        .unwrap();
2243        git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2244            .await
2245            .unwrap();
2246        source.status = RunStatus::Merged;
2247        source.save_under(&f.home).unwrap();
2248        assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
2249        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2250        assert!(
2251            crate::graph::fold_run(&mut source, true, &f.home)
2252                .await
2253                .unwrap_err()
2254                .to_string()
2255                .contains("explicitly retained")
2256        );
2257        // The fixture's explicit marker is removed only to exercise ordinary
2258        // confirmed-merge cleanup; no production cleanup removes such markers.
2259        std::fs::remove_file(
2260            f.home
2261                .join("runs")
2262                .join(&source.id)
2263                .join("retain-recovery-sources"),
2264        )
2265        .unwrap();
2266        let (folded, unreadable) = crate::clean::fold_due(
2267            &f.home.join("runs"),
2268            &f.home,
2269            &f.bay,
2270            &Fx::disk(),
2271            Fx::now(),
2272        )
2273        .await
2274        .unwrap();
2275        assert_eq!((folded, unreadable), (1, 0));
2276        assert!(!source.candidates[0].worktree.exists());
2277        let evidence = f
2278            .home
2279            .join("runs")
2280            .join(&source.id)
2281            .join("artifacts/fold-evidence");
2282        let names: Vec<_> = std::fs::read_dir(evidence)
2283            .unwrap()
2284            .map(|entry| entry.unwrap().file_name())
2285            .collect();
2286        assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
2287        assert_eq!(
2288            queue.get(&task.id).unwrap().status,
2289            crate::queue::TaskStatus::Held
2290        );
2291    }
2292
2293    #[tokio::test]
2294    async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
2295        for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
2296            let f = Fx::new().await;
2297            let mut state = f.run("cc06", true).await;
2298            let candidate = state.candidates[0].clone();
2299            let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2300            std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
2301            git::rescue_commit(&candidate.worktree, "second candidate change")
2302                .await
2303                .unwrap();
2304            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2305            state.tally = Some(crate::run::Tally {
2306                first_choice: Default::default(),
2307                borda: Default::default(),
2308                winner: 'A',
2309                rankings: 0,
2310                unanimous_initial: true,
2311                deliberated: false,
2312                changed_votes: 0,
2313                unanimous_final: true,
2314                tie_break: None,
2315                judges: 0,
2316                present: 0,
2317                quorum: 0,
2318                met_quorum: true,
2319                uncontested: Some("fixture".into()),
2320            });
2321            // A reviewer pinned to an intermediate candidate commit must also
2322            // be reclaimable, although neither commit ID lands in the squash.
2323            let reviewer = state.worktree_root().join("review-1");
2324            git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
2325                .await
2326                .unwrap();
2327            // Concurrent base changes and later base advances prevent a mere
2328            // equality check against the current base tree from passing.
2329            std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
2330            git::rescue_commit(&f.repo, "advance before squash")
2331                .await
2332                .unwrap();
2333            let published = if variant == "partial" {
2334                &intermediate
2335            } else {
2336                &head
2337            };
2338            git::git(&f.repo, &["merge", "--squash", published])
2339                .await
2340                .unwrap();
2341            git::git(&f.repo, &["commit", "-m", "squash candidate"])
2342                .await
2343                .unwrap();
2344            let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
2345            assert!(
2346                !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
2347                    .await
2348                    .unwrap()
2349                    .ok()
2350            );
2351            std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
2352            git::rescue_commit(&f.repo, "advance after squash")
2353                .await
2354                .unwrap();
2355            git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
2356                .await
2357                .unwrap();
2358            if matches!(variant, "checkpoint" | "reviewer") {
2359                // A reset hides unique work from HEAD, but not its reflog.
2360                std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
2361                git::rescue_commit(&reviewer, "unpublished checkpoint")
2362                    .await
2363                    .unwrap();
2364                if variant == "checkpoint" {
2365                    git::git(&reviewer, &["reset", "--hard", &intermediate])
2366                        .await
2367                        .unwrap();
2368                }
2369            }
2370            if variant == "loser" {
2371                let mut loser = candidate.clone();
2372                loser.label = 'B';
2373                loser.branch = "magi/cc06/B".into();
2374                loser.worktree = state.worktree_root().join("cand-B");
2375                git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
2376                    .await
2377                    .unwrap();
2378                std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
2379                git::rescue_commit(&loser.worktree, "independent loser")
2380                    .await
2381                    .unwrap();
2382                state.candidates.push(loser);
2383            }
2384            state.status = RunStatus::Merged;
2385            state.merge = Some(crate::run::MergeOutcome {
2386                mode: crate::config::MergeMode::Pr,
2387                ok: true,
2388                detail: "candidate A squash merged".into(),
2389                empty: false,
2390            });
2391            state.save_under(&f.home).unwrap();
2392            let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
2393            let result = crate::graph::fold_run(&mut state, true, &f.home).await;
2394            result.unwrap();
2395            assert!(
2396                !candidate.worktree.exists() && !reviewer.exists(),
2397                "{variant}"
2398            );
2399            assert!(
2400                !git::branch_exists(&f.repo, &candidate.branch)
2401                    .await
2402                    .unwrap()
2403            );
2404            let artifacts = f
2405                .home
2406                .join("runs")
2407                .join(&state.id)
2408                .join("artifacts/fold-evidence");
2409            assert_eq!(
2410                artifacts.join(format!("{head}.patch")).exists(),
2411                variant == "partial",
2412                "only a partially published winner needs evidence: {variant}"
2413            );
2414            assert_eq!(
2415                artifacts.join(format!("{intermediate}.patch")).exists(),
2416                variant == "partial",
2417                "squash proof also covers intermediate reviewer tips: {variant}"
2418            );
2419            let patches: Vec<_> = std::fs::read_dir(&artifacts)
2420                .unwrap()
2421                .map(|entry| entry.unwrap().path())
2422                .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
2423                .collect();
2424            let expected = match variant {
2425                "partial" => 2,
2426                "loser" | "reviewer" => 1,
2427                _ => 0,
2428            };
2429            assert_eq!(patches.len(), expected, "{variant}");
2430            if variant == "loser" {
2431                assert!(
2432                    std::fs::read_to_string(&patches[0])
2433                        .unwrap()
2434                        .contains("losing candidate")
2435                );
2436            }
2437            assert!(artifacts.join("manifest.json").exists());
2438            if variant == "reviewer" {
2439                assert!(
2440                    std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
2441                        .unwrap()
2442                        .contains("reviewer checkpoint")
2443                );
2444            }
2445            crate::graph::fold_run(&mut state, true, &f.home)
2446                .await
2447                .unwrap();
2448        }
2449    }
2450
2451    #[tokio::test]
2452    async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
2453        for missing in [true, false] {
2454            let f = Fx::new().await;
2455            let mut state = f.run("cc05", true).await;
2456            // A fetched base exists, but these local review changes were not
2457            // incorporated by ancestry, so their recovery evidence is required.
2458            git::git(
2459                &f.repo,
2460                &["update-ref", "refs/remotes/origin/main", &f.base],
2461            )
2462            .await
2463            .unwrap();
2464            let candidate = state.candidates[0].clone();
2465            if !missing {
2466                std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
2467                git::rescue_commit(&f.repo, "advance base before rebase")
2468                    .await
2469                    .unwrap();
2470                git::git(&candidate.worktree, &["rebase", "main"])
2471                    .await
2472                    .unwrap();
2473                // Match the runner's post-rebase synchronization, which
2474                // clears the merge backend's transient AUTO_MERGE checkpoint.
2475                git::sync_to_head(&candidate.worktree).await.unwrap();
2476            }
2477            let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
2478            // Model review-only state: no implementation-time snapshot exists.
2479            state.reviewed_commits = Some(Vec::new());
2480            state.candidates[0].worktree = state.worktree_root().join("under-review");
2481            git::git(
2482                &f.repo,
2483                &[
2484                    "worktree",
2485                    "move",
2486                    candidate.worktree.to_str().unwrap(),
2487                    state.candidates[0].worktree.to_str().unwrap(),
2488                ],
2489            )
2490            .await
2491            .unwrap();
2492            let reviewer = state.worktree_root().join("review-1");
2493            git::worktree_add_detached(&f.repo, &reviewer, &head)
2494                .await
2495                .unwrap();
2496            // A reset/rebase can leave unpublished recovery history. It must
2497            // not keep any remaining owned worktree alive indefinitely.
2498            std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
2499            git::rescue_commit(&reviewer, "old review history")
2500                .await
2501                .unwrap();
2502            git::git(&reviewer, &["reset", "--hard", &head])
2503                .await
2504                .unwrap();
2505            if missing {
2506                std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
2507            }
2508            let run_dir = f.home.join("runs").join(&state.id);
2509            let artifacts = run_dir.join("artifacts");
2510            std::fs::create_dir_all(&artifacts).unwrap();
2511            std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
2512            if missing {
2513                std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
2514            }
2515            state.status = RunStatus::Merged;
2516            state.merge = Some(crate::run::MergeOutcome {
2517                mode: crate::config::MergeMode::Pr,
2518                ok: true,
2519                detail: "confirmed squash merge".into(),
2520                empty: false,
2521            });
2522            state.save_under(&f.home).unwrap();
2523            // Unexpected external paths remain protected, even if absent.
2524            let owned = state.candidates[0].worktree.clone();
2525            state.candidates[0].worktree = f.home.join("missing-external");
2526            assert!(
2527                crate::graph::fold_run(&mut state, true, &f.home)
2528                    .await
2529                    .unwrap_err()
2530                    .to_string()
2531                    .contains("outside the inventoried")
2532            );
2533            state.candidates[0].worktree = owned;
2534            // A live output borrower blocks the entire preflight.
2535            let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
2536            assert!(
2537                crate::graph::fold_run(&mut state, true, &f.home)
2538                    .await
2539                    .is_err()
2540            );
2541            assert!(reviewer.exists());
2542            drop(guard);
2543            // Failure to persist evidence must not cause partial removal.
2544            let evidence = artifacts.join("fold-evidence");
2545            std::fs::write(&evidence, "blocked").unwrap();
2546            assert!(
2547                crate::graph::fold_run(&mut state, true, &f.home)
2548                    .await
2549                    .is_err()
2550            );
2551            assert!(reviewer.exists());
2552            std::fs::remove_file(&evidence).unwrap();
2553            std::fs::create_dir_all(&evidence).unwrap();
2554            let oversized = evidence.join("capacity-fixture");
2555            std::fs::File::create(&oversized)
2556                .unwrap()
2557                .set_len(64 * 1024 * 1024)
2558                .unwrap();
2559            assert!(
2560                crate::graph::fold_run(&mut state, true, &f.home)
2561                    .await
2562                    .is_err()
2563            );
2564            assert!(reviewer.exists());
2565            std::fs::remove_file(oversized).unwrap();
2566            // Ignored recovery files and unfinished Git operations still hold
2567            // all sources, even for a confirmed merge.
2568            std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
2569            std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
2570            assert!(
2571                crate::graph::fold_run(&mut state, true, &f.home)
2572                    .await
2573                    .is_err()
2574            );
2575            std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
2576            let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
2577                .await
2578                .unwrap();
2579            std::fs::write(&checkpoint, &head).unwrap();
2580            assert!(
2581                crate::graph::fold_run(&mut state, true, &f.home)
2582                    .await
2583                    .is_err()
2584            );
2585            std::fs::remove_file(checkpoint).unwrap();
2586            let (folded, unreadable) = crate::clean::fold_due(
2587                &f.home.join("runs"),
2588                &f.home,
2589                &f.bay,
2590                &Fx::disk(),
2591                Fx::now(),
2592            )
2593            .await
2594            .unwrap();
2595            if folded != 1 {
2596                panic!(
2597                    "normal cleanup failed: {:#}",
2598                    crate::graph::fold_run(&mut state, true, &f.home)
2599                        .await
2600                        .unwrap_err()
2601                );
2602            }
2603            assert_eq!((folded, unreadable), (1, 0));
2604            assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
2605            state = RunState::load_under(&state.id, &f.home).unwrap();
2606            assert!(state.candidates[0].folded);
2607            let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
2608            crate::graph::fold_run(&mut state, true, &f.home)
2609                .await
2610                .unwrap();
2611            assert_eq!(
2612                std::fs::read(evidence.join("manifest.json")).unwrap(),
2613                manifest
2614            );
2615            git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
2616                .await
2617                .unwrap();
2618            git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
2619            assert!(
2620                !git::git_raw(&f.repo, &["cat-file", "-e", &head])
2621                    .await
2622                    .unwrap()
2623                    .ok()
2624            );
2625            // Restore the modified file into a fresh repository from artifact
2626            // bytes alone; no original commit or blob is consulted.
2627            let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
2628            let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
2629                .as_str()
2630                .unwrap();
2631            let recovered = f.home.join("recovered");
2632            std::fs::create_dir_all(&recovered).unwrap();
2633            git::git(&recovered, &["init"]).await.unwrap();
2634            std::fs::write(
2635                recovered.join("source"),
2636                std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
2637            )
2638            .unwrap();
2639            git::git(
2640                &recovered,
2641                &[
2642                    "apply",
2643                    evidence.join(format!("{head}.patch")).to_str().unwrap(),
2644                ],
2645            )
2646            .await
2647            .unwrap();
2648            assert_eq!(
2649                std::fs::read_to_string(recovered.join("source")).unwrap(),
2650                "cc05"
2651            );
2652            assert!(
2653                !std::fs::read(evidence.join(format!("{head}.patch")))
2654                    .unwrap()
2655                    .is_empty()
2656            );
2657            assert_eq!(
2658                std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
2659                "review findings"
2660            );
2661            if missing {
2662                assert_eq!(
2663                    std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
2664                    "original snapshot"
2665                );
2666            } else {
2667                assert!(!artifacts.join("cand-A.patch").exists());
2668            }
2669            assert!(run_dir.join("run.json").exists());
2670        }
2671    }
2672
2673    #[tokio::test]
2674    async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
2675        use std::os::unix::fs::PermissionsExt as _;
2676        let f = Fx::new().await;
2677        let source = f.run("dd03", true).await;
2678        let empty = f.run("dd04", false).await;
2679        let task = f.task(&[source, empty.clone()]);
2680        let entered = f.repo.join("proof-entered");
2681        let release = f.repo.join("proof-release");
2682        let hook = f.repo.join("proof-monitor");
2683        std::fs::write(&hook, format!(
2684            "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
2685            entered.display(), release.display())).unwrap();
2686        std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
2687        git::git(
2688            &f.repo,
2689            &["config", "core.fsmonitor", hook.to_str().unwrap()],
2690        )
2691        .await
2692        .unwrap();
2693        struct ReleaseProof(PathBuf);
2694        impl Drop for ReleaseProof {
2695            fn drop(&mut self) {
2696                let _ = std::fs::write(&self.0, "continue");
2697            }
2698        }
2699        let _release_on_panic = ReleaseProof(release.clone());
2700        let home = f.home.clone();
2701        let sweep = tokio::spawn(async move {
2702            let mut report = Report::default();
2703            retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
2704            report
2705        });
2706        tokio::time::timeout(std::time::Duration::from_secs(5), async {
2707            while !entered.exists() {
2708                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2709            }
2710        })
2711        .await
2712        .unwrap();
2713        let queue = crate::queue::Queue::at(f.home.join("queue"));
2714        let mut selected = queue.get(&task.id).unwrap();
2715        selected.resume_override = Some(crate::queue::OperatorResume {
2716            question_id: "during-slow-proof".into(),
2717            at: Fx::now(),
2718            conductor_rehold: None,
2719            forced: false,
2720            pinned_run: Some(empty.id.clone()),
2721        });
2722        let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
2723        let quick = match tokio::time::timeout(std::time::Duration::from_secs(2), &mut update).await
2724        {
2725            Ok(result) => {
2726                result.unwrap().unwrap();
2727                true
2728            }
2729            Err(_) => {
2730                std::fs::write(&release, "continue").unwrap();
2731                update.await.unwrap().unwrap();
2732                false
2733            }
2734        };
2735        std::fs::write(release, "continue").unwrap();
2736        let report = tokio::time::timeout(std::time::Duration::from_secs(8), sweep)
2737            .await
2738            .unwrap()
2739            .unwrap();
2740        assert!(
2741            quick,
2742            "task writers must not wait behind Git retirement proof"
2743        );
2744        assert_eq!(report.retired, 0);
2745        assert!(report.retained.iter().any(|r| r.contains("task changed")));
2746        assert!(empty.candidates[0].worktree.exists());
2747        assert!(
2748            RunState::load_under(&empty.id, &f.home)
2749                .unwrap()
2750                .retired_to
2751                .is_none()
2752        );
2753    }
2754
2755    #[tokio::test]
2756    async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
2757        let f = Fx::new().await;
2758        let source = f.run("dd01", true).await;
2759        let mut empty = f.run("dd02", false).await;
2760        let expected = f.task(&[source.clone(), empty.clone()]);
2761        let queue = crate::queue::Queue::at(f.home.join("queue"));
2762        let _claim = queue.claim(&expected.id).unwrap();
2763        // The proof holds run/output leases, never the task write lock.
2764        let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
2765        let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
2766        let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
2767        let mut selected = queue.get(&expected.id).unwrap();
2768        selected.resume_override = Some(crate::queue::OperatorResume {
2769            question_id: "new-owner-decision".into(),
2770            at: Fx::now(),
2771            conductor_rehold: None,
2772            forced: false,
2773            pinned_run: Some(empty.id.clone()),
2774        });
2775        queue.put(&mut selected).unwrap();
2776        let error = commit_retirement(
2777            &f.home,
2778            &queue,
2779            &expected,
2780            &mut empty,
2781            &source.id,
2782            &expected.runs,
2783            Fx::now(),
2784        )
2785        .unwrap_err();
2786        assert!(error.to_string().contains("task changed"));
2787        assert!(
2788            RunState::load_under(&empty.id, &f.home)
2789                .unwrap()
2790                .retired_to
2791                .is_none()
2792        );
2793        assert!(empty.candidates[0].worktree.exists());
2794        assert_eq!(
2795            queue
2796                .get(&expected.id)
2797                .unwrap()
2798                .resume_override
2799                .unwrap()
2800                .pinned_run,
2801            Some(empty.id.clone())
2802        );
2803        assert!(
2804            !f.home
2805                .join("recovery-lineages")
2806                .join(format!("{}.json", expected.id))
2807                .exists()
2808        );
2809        // Re-reading the new snapshot must not bypass its explicit pin either.
2810        assert!(
2811            commit_retirement(
2812                &f.home,
2813                &queue,
2814                &selected,
2815                &mut empty,
2816                &source.id,
2817                &selected.runs,
2818                Fx::now()
2819            )
2820            .unwrap_err()
2821            .to_string()
2822            .contains("selected this recovery source")
2823        );
2824    }
2825
2826    #[tokio::test]
2827    async fn unknown_targets_cargo_owners_and_driver_resumes_are_protected() {
2828        let f = Fx::new().await;
2829        let state = f.run("cccc", false).await;
2830        let wt = &state.candidates[0].worktree;
2831        Fx::target(wt);
2832        std::fs::write(wt.join("target/notes"), "user file").unwrap();
2833        assert!(cargo_target(wt).await.is_err());
2834        std::fs::remove_file(wt.join("target/notes")).unwrap();
2835        std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
2836        assert!(
2837            !declared_outputs(&wt.join("target"))
2838                .unwrap()
2839                .contains(&wt.join("target/debug/link"))
2840        );
2841        std::fs::remove_file(wt.join("target/debug/link")).unwrap();
2842        let cargo = cargo_locks(&wt.join("target")).unwrap();
2843        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2844        assert_eq!(out.freed_bytes, 0);
2845        assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
2846        drop(cargo);
2847        let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2848        let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2849        assert_eq!(out.freed_bytes, 0);
2850        drop(driver);
2851        let exclusive = OutputLease::take(wt, true).unwrap();
2852        assert!(
2853            OutputLease::borrow(wt).is_err(),
2854            "resume cannot start a writer during cleanup"
2855        );
2856        drop(exclusive);
2857        assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
2858    }
2859
2860    #[tokio::test]
2861    async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
2862        let f = Fx::new().await;
2863        let source = f.run("0000", true).await;
2864        let empty = f.run("0001", false).await;
2865        let changed = f.run("0002", false).await;
2866        let unknown = f.run("0003", false).await;
2867        let unproven = f.run("0004", false).await;
2868        f.task(&[
2869            source.clone(),
2870            empty.clone(),
2871            changed.clone(),
2872            unknown.clone(),
2873            unproven.clone(),
2874        ]);
2875        for run in [&empty, &changed, &unknown] {
2876            Fx::target(&run.candidates[0].worktree);
2877        }
2878        let reviewer = empty.worktree_root().join("review-1");
2879        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
2880            .await
2881            .unwrap();
2882        Fx::target(&reviewer);
2883        let mut disk = Fx::disk();
2884        disk.auto_fold = false;
2885        let report = sweep(&f.home, &disk, Fx::now()).await;
2886        assert_eq!(report.freed_bytes, 800, "{report:?}");
2887        let wt = &empty.candidates[0].worktree;
2888        assert!(wt.join("target/debug/output.d").exists());
2889        assert!(!wt.join("target/debug/output").exists());
2890        let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
2891        let proof: CargoResidue =
2892            serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
2893        assert_eq!(
2894            proof.files[Path::new("debug/output.d")],
2895            std::fs::read(wt.join("target/debug/output.d")).unwrap()
2896        );
2897        std::fs::write(
2898            changed.candidates[0].worktree.join("target/debug/output.d"),
2899            "owner notes",
2900        )
2901        .unwrap();
2902        std::fs::write(
2903            unknown.candidates[0].worktree.join("target/debug/notes"),
2904            "keep",
2905        )
2906        .unwrap();
2907        Fx::target(&unproven.candidates[0].worktree);
2908        std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
2909        disk.auto_fold = true;
2910        let locks = cargo_locks(&wt.join("target")).unwrap();
2911        let mut report = Report::default();
2912        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2913        assert_eq!(report.retired, 0, "{report:?}");
2914        assert!(
2915            report
2916                .retained
2917                .iter()
2918                .any(|reason| reason.contains("Cargo build"))
2919        );
2920        drop(locks);
2921        retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
2922        assert_eq!(report.retired, 1, "{report:?}");
2923        assert!(!wt.exists());
2924        assert!(!reviewer.exists());
2925        assert!(
2926            residue_proof(&f.home, &empty.id, &reviewer)
2927                .unwrap()
2928                .exists()
2929        );
2930        assert!(
2931            proof_path.exists(),
2932            "metadata evidence outlives the worktree"
2933        );
2934        assert_eq!(
2935            RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
2936            Some(source.id)
2937        );
2938        for run in [&changed, &unknown, &unproven] {
2939            assert!(run.candidates[0].worktree.exists());
2940            assert!(
2941                RunState::load_under(&run.id, &f.home)
2942                    .unwrap()
2943                    .retired_to
2944                    .is_none()
2945            );
2946        }
2947    }
2948
2949    #[tokio::test]
2950    async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
2951        let f = Fx::new().await;
2952        let source = f.run("0000", true).await;
2953        let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
2954            .await
2955            .unwrap();
2956        let mut runs = vec![source.clone()];
2957        for n in 1..=12 {
2958            runs.push(f.run(&format!("{n:04}"), false).await);
2959        }
2960        let dirty = runs[1].candidates[0].worktree.clone();
2961        std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
2962        let ignored = runs[2].candidates[0].worktree.clone();
2963        std::fs::create_dir(ignored.join("scratch")).unwrap();
2964        std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
2965        // A missing candidate array does not prove that an unrecorded worktree
2966        // is empty: discovery must inspect Git and the worktree itself.
2967        runs[1].candidates.clear();
2968        runs[1].save_under(&f.home).unwrap();
2969        let task = f.task(&runs);
2970        let mut total = 0;
2971        for _ in 0..4 {
2972            let mut report = Report::default();
2973            retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
2974            assert!(report.retired <= 5);
2975            total += report.retired;
2976        }
2977        assert_eq!(total, 10);
2978        assert!(dirty.exists());
2979        assert!(ignored.exists());
2980        assert_eq!(
2981            git::rev_parse(&f.repo, &source.candidates[0].branch)
2982                .await
2983                .unwrap(),
2984            source_head
2985        );
2986        let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
2987        assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
2988        assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
2989        let mut released = task.clone();
2990        released.release();
2991        assert_eq!(
2992            crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
2993                RunState::load_under(id, &f.home)
2994            }),
2995            Some(source.id.clone())
2996        );
2997
2998        assert!(
2999            f.home
3000                .join("runs")
3001                .join(&retired.id)
3002                .join("run.json")
3003                .exists()
3004        );
3005        assert!(
3006            f.home
3007                .join("recovery-lineages")
3008                .join(format!("{}.json", task.id))
3009                .exists()
3010        );
3011    }
3012
3013    #[tokio::test]
3014    async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
3015        let f = Fx::new().await;
3016        let a = f.run("dddd", true).await;
3017        let b = f.run("eeee", true).await;
3018        let empty = f.run("ffff", false).await;
3019        let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
3020        assert!(
3021            crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
3022            "unfinished queue history protects older source branches from ordinary folding"
3023        );
3024        let mut prior = a.clone();
3025        assert!(
3026            crate::graph::fold_run(&mut prior, true, &f.home)
3027                .await
3028                .unwrap_err()
3029                .to_string()
3030                .contains(&task.id)
3031        );
3032        let mut report = Report::default();
3033        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3034        assert_eq!(report.retired, 0);
3035        assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
3036        crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3037        Fx::target(&a.candidates[0].worktree);
3038        let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3039        assert!(
3040            report.freed_bytes > 0,
3041            "source retention does not retain regenerable outputs"
3042        );
3043        assert!(a.candidates[0].worktree.exists());
3044        assert!(b.candidates[0].worktree.exists());
3045        assert!(empty.candidates[0].worktree.exists());
3046    }
3047
3048    #[tokio::test]
3049    async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
3050        let f = Fx::new().await;
3051        let authority = f.run("p000", true).await;
3052        let question_run = f.run("p001", false).await;
3053        let mut checkpoint = f.run("p002", false).await;
3054        checkpoint.land_approval = Some(crate::run::LandApproval {
3055            question: "pending".into(),
3056            head: f.base.clone(),
3057        });
3058        checkpoint.save_under(&f.home).unwrap();
3059        let retained = f.run("p003", false).await;
3060        std::fs::write(
3061            f.home
3062                .join("runs")
3063                .join(&retained.id)
3064                .join("retain-recovery-sources"),
3065            "owner",
3066        )
3067        .unwrap();
3068        let mut partial = f.run("p004", false).await;
3069        let reviewer = partial.worktree_root().join("review-1");
3070        git::worktree_add_detached(&f.repo, &reviewer, &f.base)
3071            .await
3072            .unwrap();
3073        // Model a crash after the retirement fence but only one removal.
3074        partial.retired_to = Some(authority.id.clone());
3075        partial.save_under(&f.home).unwrap();
3076        assert!(
3077            git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
3078                .await
3079                .unwrap()
3080        );
3081        let git_checkpoint = f.run("p005", false).await;
3082        let location = git::git(
3083            &git_checkpoint.candidates[0].worktree,
3084            &[
3085                "rev-parse",
3086                "--path-format=absolute",
3087                "--git-path",
3088                "sequencer",
3089            ],
3090        )
3091        .await
3092        .unwrap();
3093        std::fs::create_dir(location.trim()).unwrap();
3094        let mut preservation = f.run("p006", false).await;
3095        preservation
3096            .candidate_preservation
3097            .insert('A', "pending preservation".into());
3098        preservation.save_under(&f.home).unwrap();
3099        let task = f.task(&[
3100            preservation.clone(),
3101            git_checkpoint.clone(),
3102            authority.clone(),
3103            question_run.clone(),
3104            checkpoint.clone(),
3105            retained.clone(),
3106            partial.clone(),
3107        ]);
3108        let questions = crate::ask::Questions::at(f.home.join("questions"));
3109        let mut question = crate::ask::Question::new(
3110            question_run.id.clone(),
3111            "implement".into(),
3112            "fixture".into(),
3113            "owner decision".into(),
3114            String::new(),
3115            vec!["keep".into()],
3116        );
3117        questions.put(&mut question).unwrap();
3118        let mut report = Report::default();
3119        retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3120        assert_eq!(report.retired, 1, "{report:?}");
3121        assert!(!reviewer.exists());
3122        for state in [
3123            &question_run,
3124            &checkpoint,
3125            &retained,
3126            &git_checkpoint,
3127            &preservation,
3128        ] {
3129            assert!(state.candidates[0].worktree.exists());
3130            assert!(
3131                RunState::load_under(&state.id, &f.home)
3132                    .unwrap()
3133                    .retired_to
3134                    .is_none()
3135            );
3136        }
3137        assert!(questions.get(&question.id).unwrap().status.open());
3138        assert_eq!(
3139            crate::queue::Queue::at(f.home.join("queue"))
3140                .get(&task.id)
3141                .unwrap()
3142                .status,
3143            task.status
3144        );
3145    }
3146
3147    #[tokio::test]
3148    async fn preservation_schema_migration_remains_distinct_from_retirement() {
3149        let f = Fx::new().await;
3150        let mut state = f.run("p007", false).await;
3151        state.schema = 15;
3152        state.save_under(&f.home).unwrap();
3153        let native = RunState::load_under(&state.id, &f.home).unwrap();
3154        assert_eq!(native.schema, crate::run::SCHEMA);
3155        assert!(native.candidate_preservation.is_empty());
3156        assert!(native.retired_to.is_none());
3157        state.schema = 14;
3158        state.save_under(&f.home).unwrap();
3159        let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
3160        assert!(legacy.candidate_preservation.contains_key(&'A'));
3161        legacy.save_under(&f.home).unwrap();
3162        let restored = RunState::load_under(&state.id, &f.home).unwrap();
3163        assert_eq!(
3164            restored.candidate_preservation,
3165            legacy.candidate_preservation
3166        );
3167    }
3168
3169    #[tokio::test]
3170    async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
3171        let f = Fx::new().await;
3172        let cache = f.repo.join("cache");
3173        let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
3174        std::fs::create_dir_all(&cache).unwrap();
3175        assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3176        assert!(
3177            crate::cache::maintenance_prune(&f.home, &cache, 1)
3178                .unwrap()
3179                .is_none()
3180        );
3181        drop(guard);
3182        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3183    }
3184
3185    #[tokio::test]
3186    async fn launcher_death_does_not_release_live_tool_output_reservations() {
3187        let f = Fx::new().await;
3188        let run = f.run("abcd", false).await;
3189        let wt = &run.candidates[0].worktree;
3190        let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
3191        helper
3192            .quiet()
3193            .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
3194            .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
3195            .env("MAGI_TEST_WORKTREE", wt)
3196            .stdin(std::process::Stdio::null())
3197            .stdout(std::process::Stdio::null())
3198            .stderr(std::process::Stdio::null())
3199            .kill_on_drop(true);
3200        assert!(
3201            tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
3202                .await
3203                .unwrap()
3204                .unwrap()
3205                .success()
3206        );
3207        let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
3208            .unwrap()
3209            .trim()
3210            .parse()
3211            .unwrap();
3212        struct KillTool(i32);
3213        impl Drop for KillTool {
3214            fn drop(&mut self) {
3215                // SAFETY: PID belongs to the bounded helper above.
3216                unsafe { libc::kill(self.0, libc::SIGKILL) };
3217            }
3218        }
3219        let tool = KillTool(pid);
3220        assert!(OutputLease::take(wt, true).is_err());
3221        assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
3222        drop(tool);
3223        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3224            loop {
3225                if OutputLease::take(wt, true).is_ok()
3226                    && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
3227                {
3228                    break;
3229                }
3230                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3231            }
3232        })
3233        .await
3234        .unwrap();
3235    }
3236
3237    #[tokio::test]
3238    async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
3239        let f = Fx::new().await;
3240        let run = f.run("abcd", false).await;
3241        let wt = &run.candidates[0].worktree;
3242        let cache = f.repo.join("cache");
3243        std::fs::create_dir(&cache).unwrap();
3244        for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
3245            let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
3246            let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
3247            cmd.quiet()
3248                .args(["--exact", "seat_process::tests::orphan_tool_helper"])
3249                .env("MAGI_TEST_ORPHAN_PID", &pid_file)
3250                .stdin(std::process::Stdio::null())
3251                .stdout(std::process::Stdio::null())
3252                .stderr(std::process::Stdio::null())
3253                .kill_on_drop(true);
3254            if clear_owner {
3255                cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
3256            } else {
3257                cmd.env("MAGI_TEST_CLOSE_FDS", "1");
3258            }
3259            let mut running = crate::seat_process::Running::spawn(
3260                cmd,
3261                None,
3262                vec![
3263                    OutputLease::borrow(wt).unwrap().unwrap(),
3264                    OutputLease::cache(&f.home, &cache, false).unwrap(),
3265                ],
3266            )
3267            .unwrap();
3268            assert!(
3269                tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
3270                    .await
3271                    .unwrap()
3272                    .unwrap()
3273                    .success()
3274            );
3275            let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
3276            struct KillTool(Option<i32>);
3277            impl Drop for KillTool {
3278                fn drop(&mut self) {
3279                    // SAFETY: PID from our helper; kill also bounds panic paths.
3280                    if let Some(pid) = self.0 {
3281                        unsafe { libc::kill(pid, libc::SIGKILL) };
3282                    }
3283                }
3284            }
3285            let mut tool = KillTool(Some(pid));
3286            // The CLI has already exited and cannot supply a parent-tree
3287            // relationship. Either the witness or inherited environment must
3288            // retain ownership; test each independently.
3289            assert!(OutputLease::take(wt, true).is_err());
3290            let finish = if cancel {
3291                drop(running);
3292                None
3293            } else {
3294                Some(tokio::spawn(async move { running.finish().await }))
3295            };
3296            if clear_owner {
3297                tokio::time::sleep(std::time::Duration::from_millis(150)).await;
3298                assert!(crate::proc::pid_alive(pid as u32));
3299                assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
3300                assert!(OutputLease::cache(&f.home, &cache, true).is_err());
3301                assert!(
3302                    crate::cache::maintenance_prune(&f.home, &cache, 0)
3303                        .unwrap()
3304                        .is_none()
3305                );
3306                if let Some(finish) = &finish {
3307                    assert!(!finish.is_finished());
3308                }
3309                // Witness-only ownership waits for natural/external tool exit.
3310                unsafe { libc::kill(pid, libc::SIGKILL) };
3311            }
3312            if let Some(finish) = finish {
3313                tokio::time::timeout(std::time::Duration::from_secs(5), finish)
3314                    .await
3315                    .unwrap()
3316                    .unwrap();
3317            }
3318            tokio::time::timeout(std::time::Duration::from_secs(5), async {
3319                loop {
3320                    if OutputLease::take(wt, true).is_ok()
3321                        && OutputLease::cache(&f.home, &cache, true).is_ok()
3322                    {
3323                        // Token-owned orphans must be stopped by cleanup itself.
3324                        if !clear_owner {
3325                            assert!(!crate::proc::pid_alive(pid as u32));
3326                        }
3327                        break;
3328                    }
3329                    tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3330                }
3331            })
3332            .await
3333            .unwrap();
3334            tool.0 = None;
3335        }
3336    }
3337
3338    #[tokio::test]
3339    async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
3340        let f = Fx::new().await;
3341        let run = f.run("abcd", false).await;
3342        let wt = &run.candidates[0].worktree;
3343        let cache = f.repo.join("cache");
3344        std::fs::create_dir(&cache).unwrap();
3345        let pid_file = f.repo.join("child-pid");
3346        let mut cmd = tokio::process::Command::new("sh");
3347        cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
3348            .quiet()
3349            .env("CHILD_PID_FILE", &pid_file)
3350            .process_group(0)
3351            .kill_on_drop(true);
3352        let running = crate::seat_process::Running::spawn(
3353            cmd,
3354            None,
3355            vec![
3356                OutputLease::borrow(wt).unwrap().unwrap(),
3357                OutputLease::cache(&f.home, &cache, false).unwrap(),
3358            ],
3359        )
3360        .unwrap();
3361        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3362            while !pid_file.exists() {
3363                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3364            }
3365        })
3366        .await
3367        .unwrap();
3368        assert!(OutputLease::take(wt, true).is_err());
3369        assert!(
3370            crate::cache::maintenance_prune(&f.home, &cache, 0)
3371                .unwrap()
3372                .is_none()
3373        );
3374        let pid: u32 = std::fs::read_to_string(pid_file)
3375            .unwrap()
3376            .trim()
3377            .parse()
3378            .unwrap();
3379        drop(running);
3380        tokio::time::timeout(std::time::Duration::from_secs(10), async {
3381            loop {
3382                if let Ok(guard) = OutputLease::take(wt, true) {
3383                    assert!(
3384                        !crate::proc::pid_alive(pid),
3385                        "output lock released before tool death"
3386                    );
3387                    drop(guard);
3388                    break;
3389                }
3390                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
3391            }
3392        })
3393        .await
3394        .unwrap();
3395        assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
3396    }
3397
3398    #[tokio::test]
3399    async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
3400        let f = Fx::new().await;
3401        let reason = tokio::time::timeout(
3402            std::time::Duration::from_secs(2),
3403            emergency(vec![f.repo.clone()], u64::MAX),
3404        )
3405        .await
3406        .unwrap();
3407        assert!(reason.contains("disk pressure"));
3408        assert!(
3409            tokio::time::timeout(
3410                std::time::Duration::from_millis(10),
3411                emergency(vec![f.repo.clone()], 0)
3412            )
3413            .await
3414            .is_err()
3415        );
3416        assert_eq!(
3417            std::fs::read_to_string(f.repo.join("source")).unwrap(),
3418            "base\n"
3419        );
3420    }
3421
3422    #[tokio::test]
3423    async fn automatic_requeues_cannot_refund_failed_input_generation() {
3424        let f = Fx::new().await;
3425        let mut run = f.run("1234", false).await;
3426        run.prep_failure = Some("cherry-pick conflict".into());
3427        let mut task = f.task(std::slice::from_ref(&run));
3428        let inputs = crate::queue::recovery_inputs(&run.sources);
3429        for _ in 0..108 {
3430            task.recovery_failed(&run);
3431            task.requeue();
3432            assert_eq!(task.attempts, 0);
3433        }
3434        let bytes = serde_json::to_vec(&task).unwrap();
3435        let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
3436        assert!(
3437            restored
3438                .recovery_exhausted(&run.base_commit, &inputs, 2)
3439                .is_some()
3440        );
3441        assert!(
3442            restored
3443                .recovery_exhausted("different-base", &inputs, 2)
3444                .is_none()
3445        );
3446        let original_base = run.base_commit.clone();
3447        run.base_commit = "different-base".into();
3448        restored.recovery_failed(&run);
3449        assert!(
3450            restored
3451                .recovery_exhausted(&original_base, &inputs, 2)
3452                .is_some(),
3453            "returning to old inputs does not replenish their budget"
3454        );
3455        assert!(
3456            restored
3457                .recovery_exhausted(&run.base_commit, &inputs, 2)
3458                .is_none()
3459        );
3460        restored.release_operator("test-owner", None);
3461        assert!(
3462            restored
3463                .recovery_exhausted(&run.base_commit, &inputs, 2)
3464                .is_none()
3465        );
3466    }
3467}