Skip to main content

magi/
clean.rs

1//! The disk janitor: finished runs get their worktrees folded, worktrees whose
2//! run record is already gone get reclaimed too, and the shared build cache is
3//! pruned to its cap.
4//!
5//! A run's state being written by an older schema is not the same thing as it
6//! being unreadable, and this module used to conflate the two: [`fold_due`]
7//! treated any `run.json` its version check rejected exactly like one that
8//! failed to parse at all, so a single schema bump silently stopped every
9//! automatic fold in the fleet the moment it shipped, and did so with no
10//! counter and no log line to say so. A record magi genuinely cannot parse —
11//! missing fields, broken JSON, a schema newer than this build has ever heard
12//! of — is still left alone here, still counted in
13//! [`Housekeeping::unreadable`], and still only ever removed by an explicit
14//! operator action (`magi fold`, or the equivalent phone route). One written
15//! by a schema this build merely disagrees with the *meaning* of is not that:
16//! as long as it still parses, folding proceeds regardless of the number in
17//! its `schema` field.
18//!
19//! Everything policy-shaped — which statuses are foldable, how long a finished
20//! run is left alone, whether the cache is over its limit — is a pure function
21//! injected with numbers, so nothing here has to ask the operating system to
22//! be testable. The only I/O is the removal itself.
23
24use std::collections::BTreeSet;
25use std::path::{Path, PathBuf};
26
27use anyhow::{Context as _, Result, bail};
28use jiff::{SignedDuration, Timestamp};
29use serde::Deserialize;
30
31use crate::ask::Questions;
32use crate::config::Disk;
33use crate::run::{RunState, RunStatus, SCHEMA, short_of};
34
35use crate::disk::{Prune, dir_size};
36
37/// What one janitor pass did, for the caller's log line.
38#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
39pub struct Housekeeping {
40    /// Runs folded (worktrees dropped).
41    pub folded: usize,
42    /// Runs [`fold_due`] left alone because their `run.json` genuinely could
43    /// not be read - missing, broken JSON, a schema this build has never
44    /// heard of - as opposed to one merely written by a different schema
45    /// number, which is folded like any other (see the module docs). This was
46    /// defined but never incremented for a long stretch of this module's
47    /// history, which is exactly how 90 of 93 runs sat unfolded on one
48    /// operator's machine with nothing anywhere saying why: every one of them
49    /// was misclassified as unreadable by a schema check that has since been
50    /// narrowed to only the runs that actually are.
51    pub unreadable: usize,
52    /// Worktrees under the worktree bay reclaimed because no run record in
53    /// `runs/` claims them anymore (see [`fold_orphaned_worktrees`]).
54    pub orphaned_worktrees: usize,
55    /// Files dropped from the shared cache.
56    pub cache_files: usize,
57    /// Bytes freed from the shared cache.
58    pub cache_freed: u64,
59    /// Open questions abandoned because the run that asked them has already
60    /// settled where nothing is coming back to read an answer.
61    pub questions_abandoned: usize,
62    /// Runs found blocked on a merge magi never recorded — a pull request the
63    /// operator merged by hand while `land::land` never got as far as
64    /// opening one itself — and corrected automatically. See
65    /// [`reconcile_external_merges`].
66    pub external_merges_recorded: usize,
67    /// Run records that still called their pull request open after it was
68    /// merged or closed, and were rewritten (see
69    /// [`crate::land::repair_stale_pr_states`]).
70    pub stale_pr_states_repaired: usize,
71}
72
73/// Run the janitor: fold due runs, reclaim orphaned worktrees, prune stale
74/// worktree registrations, then prune the cache if it is over its cap.
75///
76/// Every part is best-effort; a jammed cache lock or a run whose worktree
77/// another borrower holds must not stop the rest. Errors are reported through
78/// `tracing::warn` - this is housekeeping, and the daemon keeps serving
79/// either way.
80pub async fn housekeep(
81    cfg: &crate::config::Config,
82    home: &Path,
83    worktrees_root: &Path,
84    repo: &Path,
85    now: Timestamp,
86) -> Housekeeping {
87    let mut out = Housekeeping::default();
88    crate::retention::sweep(home, &cfg.disk, now).await;
89    if cfg.disk.auto_fold {
90        let runs = home.join("runs");
91        match fold_due(&runs, home, worktrees_root, &cfg.disk, now).await {
92            Ok((folded, unreadable)) => {
93                out.folded = folded;
94                out.unreadable = unreadable;
95            }
96            Err(e) => {
97                tracing::warn!("housekeep: fold due runs: {e:#}");
98                // Stable wording: the reason varies per pass, and a changed
99                // message would relight the bell on every retry.
100                crate::notices::raise_in(
101                    home,
102                    crate::notices::Notice::warn(
103                        "housekeep:fold",
104                        "Automatic cleanup of finished runs failed; disk usage may keep growing.",
105                    ),
106                );
107            }
108        }
109        out.orphaned_worktrees =
110            fold_orphaned_worktrees(&runs, worktrees_root, home, cfg.disk.fold_grace_secs, now)
111                .await;
112        // Best-effort in the same sense as everything else here: a repository
113        // this janitor pass has nothing to do with (or none at all, in a unit
114        // test) must not turn a `warn` into a reason to skip the rest.
115        if let Err(e) = crate::git::worktree_prune(repo).await {
116            tracing::warn!("housekeep: prune worktree registrations: {e:#}");
117        }
118        out.external_merges_recorded = reconcile_external_merges(&runs, home, &cfg.disk, now).await;
119        // Bounded: a handful of forge lookups a pass keeps this cheap, and the
120        // pass is idempotent, so a backlog drains over several passes.
121        out.stale_pr_states_repaired = crate::land::repair_stale_pr_states(home, 5).await.0;
122    }
123    match prune_cache_if_over_limit(cfg, home) {
124        Ok(Some(pruned)) => {
125            out.cache_files = pruned.files;
126            out.cache_freed = pruned.freed;
127        }
128        Ok(None) => {}
129        Err(e) => tracing::warn!("housekeep: prune cache: {e:#}"),
130    }
131    match crate::build_target::prune_registered(home, &cfg.disk) {
132        Ok(pruned) => {
133            out.cache_files = out.cache_files.saturating_add(pruned.files);
134            out.cache_freed = out.cache_freed.saturating_add(pruned.freed);
135        }
136        Err(e) => tracing::warn!("housekeep: prune isolated builds: {e:#}"),
137    }
138    // Unconditional, unlike the two passes above: this is not a disk policy
139    // with a cap or an opt-out, it is closing a gap `graph::Runner` itself
140    // cannot - a run that reached `Merged`/`Ready`/`Failed` before this
141    // cleanup existed, or whose process died between saving that status and
142    // abandoning the question it leaves behind (see `Runner::settle_questions`).
143    // Left alone, that question sits `open` forever: the owner's badge,
144    // banner and title all keep counting a decision nobody is left to read.
145    out.questions_abandoned =
146        abandon_settled_questions(&Questions::at(home.join("questions")), &home.join("runs"));
147    out
148}
149
150/// Abandon every open question whose run has already settled into a status
151/// nothing comes back from, worded with what the run became - the same
152/// cleanup `graph::Runner::settle_questions` runs the moment `status` lands
153/// there, for questions that missed it.
154///
155/// Scans questions rather than runs: the open list is normally short, and a
156/// run that never asked anything costs nothing here. A run this cannot read,
157/// deleted or written by a schema this build does not speak, is left alone
158/// the same as everywhere else in this module; the question stays open
159/// rather than guessed at.
160pub fn abandon_settled_questions(store: &Questions, runs: &Path) -> usize {
161    let waiting_on: BTreeSet<String> = store
162        .list()
163        .into_iter()
164        .filter(|q| q.status.open())
165        .map(|q| q.run)
166        .collect();
167    let mut abandoned = 0;
168    for run in waiting_on {
169        let Ok(meta) = read_meta(runs, &run) else {
170            continue;
171        };
172        match store.settle_run(&run, meta.status) {
173            Ok(n) => abandoned += n,
174            Err(e) => tracing::warn!("housekeep: abandon questions for {run}: {e:#}"),
175        }
176    }
177    abandoned
178}
179
180/// Fold every run that is finished, older than the grace period, and not being
181/// worked on; return `(folded, unreadable)`.
182///
183/// A run whose `run.json` genuinely cannot be parsed — missing fields, broken
184/// JSON, a schema newer than this build has ever heard of — is left exactly
185/// as it is. Automatic housekeeping cannot tell a mid-write file from one that
186/// will never parse again, and `<home>/runs/<id>/` is the evidence `magi
187/// stats` and the deck read; when unsure whether it is safe to touch, the
188/// janitor keeps rather than deletes (see the module docs). Discarding a
189/// record this unreadable is an explicit operator action (`magi fold`, or the
190/// equivalent phone route), never something that happens unattended. Every
191/// such skip is counted in the returned `unreadable` and logged through
192/// `tracing::warn` with the parse failure that caused it - silence here is
193/// exactly the failure mode that let 90 of 93 runs sit unfolded with nothing
194/// to show for it.
195///
196/// A run merely written by a *different* schema number is not unreadable: as
197/// long as `run.json` still parses, it folds like any other terminal run (see
198/// the module docs for why the two are different questions).
199///
200/// Runnable statuses and runs newer than the grace period are also left
201/// alone; folding them would throw away work that is still the answer to
202/// somebody's question. `Merged` runs forget their winner's worktree (the
203/// merge already landed it); `Ready` and `Failed` runs keep it.
204pub async fn fold_due(
205    runs: &Path,
206    home: &Path,
207    _worktrees_root: &Path,
208    disk: &Disk,
209    now: Timestamp,
210) -> Result<(usize, usize)> {
211    let mut folded = 0usize;
212    let mut unreadable = 0usize;
213    let mut ids: Vec<String> = std::fs::read_dir(runs)
214        .into_iter()
215        .flatten()
216        .flatten()
217        .filter(|e| e.path().join("run.json").is_file())
218        .map(|e| e.file_name().to_string_lossy().into_owned())
219        .collect();
220    ids.sort_unstable();
221    for id in ids {
222        if crate::daemon::is_working_on(home, &id, now) {
223            continue;
224        }
225        let meta = match read_meta(runs, &id) {
226            Ok(meta) => meta,
227            Err(e) => {
228                unreadable += 1;
229                tracing::warn!("housekeep: run {id} unreadable, left alone: {e:#}");
230                continue;
231            }
232        };
233        if meta.status.resumable() || !due(now, meta.updated_at, disk.fold_grace_secs) {
234            continue;
235        }
236        // `read_meta` already proved the file parses; `read_state` asks for
237        // the rest of the fields `graph::fold_run` needs (worktree paths,
238        // candidates, tally). A schema mismatch alone does not fail this -
239        // see the module docs - so reaching `Err` here means the JSON itself
240        // is broken in a way `read_meta` did not exercise, which is rare but
241        // not impossible (a body truncated between the two fields it reads
242        // and the rest). That must not cost every other run its turn through
243        // this loop, so it is a skip, not a `?`.
244        let mut state = match read_state(runs, &id) {
245            Ok(state) => state,
246            Err(e) => {
247                unreadable += 1;
248                tracing::warn!("housekeep: run {id} unreadable, left alone: {e:#}");
249                continue;
250            }
251        };
252        if state.startup.as_ref().is_some_and(|s| !s.complete) {
253            continue;
254        }
255        if let Some(reason) = crate::pr_recovery::source_retention_reason(home, &state) {
256            tracing::info!(%id, %reason, "housekeep: source retained; fold refused");
257            continue;
258        }
259        if state.schema != SCHEMA {
260            tracing::info!(
261                "housekeep: run {id} was written by schema {} (this build speaks {SCHEMA}); \
262                 folding it anyway",
263                state.schema
264            );
265        }
266        // `Merged`'s winner branch is safe to drop because it already landed
267        // in the real target branch; `Superseded`'s winner branch is safe to
268        // drop for a different reason - a later attempt at the same task is
269        // what actually landed (or didn't), so *this* run's own winner is
270        // never going to be merged by anyone. Every other terminal status
271        // keeps the winner: `Ready`/`Blocked`/`Stalled`/`Failed`/`VerifiedNoop`
272        // may still have a human's decision pending on that exact branch.
273        // `AlreadyInBase` is the same: the change is on the base under other
274        // commit ids, so the branch holds nothing anyone still needs.
275        let drop_winner = matches!(
276            state.status,
277            RunStatus::Merged | RunStatus::Superseded | RunStatus::AlreadyInBase
278        );
279        // One run's fold must not cost every later run its turn. A worktree
280        // another borrower holds, a branch git refuses to delete, a repository
281        // that has since moved: each is a reason this run cannot be folded
282        // now, and none is a reason to stop the pass. Left unfolded, it is
283        // simply due again next time; a `?` here stopped automatic folding
284        // permanently at the first such run (finding R3-1-1 of run 51a3).
285        match crate::graph::fold_run(&mut state, drop_winner, home).await {
286            Ok(_) => folded += 1,
287            Err(e) => tracing::warn!("housekeep: fold {id}: {e:#}"),
288        }
289    }
290    Ok((folded, unreadable))
291}
292
293/// What one [`fetch_origins`] pass did.
294#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
295pub struct FetchReport {
296    /// Repos whose fetch succeeded.
297    pub fetched: usize,
298    /// Repos with no `origin` remote, skipped.
299    pub no_origin: usize,
300    /// Repos whose fetch failed or timed out.
301    pub failed: usize,
302}
303
304/// Run `git fetch origin` in every checkout under `roots` (the set `magi
305/// repos` lists), so `origin/main` stays current.
306///
307/// Fetch only: never a checkout, merge or fast-forward, so no HEAD, branch or
308/// working tree moves. Best-effort like [`fold_due`]: a repo with no `origin`
309/// is skipped silently, a failure or timeout is a warning, and the pass never
310/// errors. `stop` is checked between repos.
311pub async fn fetch_origins(
312    roots: &[PathBuf],
313    per_repo: std::time::Duration,
314    stop: impl Fn() -> bool,
315) -> FetchReport {
316    let mut report = FetchReport::default();
317    for repo in crate::repos::scan(roots) {
318        if stop() {
319            break;
320        }
321        match crate::git::git_raw(&repo.path, &["remote", "get-url", "origin"]).await {
322            Ok(out) if out.ok() => {}
323            _ => {
324                report.no_origin += 1;
325                continue;
326            }
327        }
328        match crate::git::fetch_origin(&repo.path, per_repo).await {
329            Ok(()) => report.fetched += 1,
330            Err(e) => {
331                tracing::warn!("fetch origin in {}: {e:#}", repo.name);
332                report.failed += 1;
333            }
334        }
335    }
336    report
337}
338
339/// Is `updated` old enough, measured against `now`, that the run may fold?
340///
341/// Pure; the janitor compares against wallclock, tests inject both sides. The
342/// comparison is strict, so a run exactly at the edge of its grace period is
343/// left alone one more pass — the same convention as [`crate::disk::over_limit`].
344pub fn due(now: Timestamp, updated: Timestamp, grace_secs: u64) -> bool {
345    now.duration_since(updated) > SignedDuration::new(grace_secs as i64, 0)
346}
347
348/// The two fields the janitor decides on, read with a serde that tolerates
349/// everything else about the run being unreadable.
350#[derive(Deserialize)]
351struct Meta {
352    status: RunStatus,
353    updated_at: Timestamp,
354}
355
356/// Read `status` and `updated_at` straight off the state file, asking for
357/// nothing else. `Err` when the file is missing, not parseable, or a status in
358/// a version this build does not speak - all of which mean "unreadable".
359fn read_meta(runs: &Path, id: &str) -> Result<Meta> {
360    let path = runs.join(id).join("run.json");
361    let body =
362        std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
363    let meta: Meta =
364        serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
365    Ok(meta)
366}
367
368/// Runs checked against GitHub in one janitor pass, at most.
369///
370/// A fleet with many stuck runs must not turn every idle tick into a burst of
371/// `gh pr list` calls; a run left unchecked this pass is checked again next
372/// pass, same as an unfolded due run is.
373const MAX_EXTERNAL_MERGE_CHECKS_PER_PASS: usize = 5;
374
375/// The fields [`reconcile_external_merges`] filters on before paying for a
376/// full [`RunState`] parse or a `gh` round trip.
377#[derive(Deserialize)]
378struct ExternalMergeMeta {
379    status: RunStatus,
380    updated_at: Timestamp,
381    #[serde(default)]
382    merge: Option<crate::run::MergeOutcome>,
383}
384
385fn read_external_merge_meta(runs: &Path, id: &str) -> Result<ExternalMergeMeta> {
386    let path = runs.join(id).join("run.json");
387    let body =
388        std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
389    let meta: ExternalMergeMeta =
390        serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
391    Ok(meta)
392}
393
394/// Is a run worth asking GitHub about?
395///
396/// Pure, so the filter itself is assertable without a fixture that writes
397/// `run.json` files or spawns `gh`. Narrower than "blocked": `merge_recorded`
398/// excludes a run `land::land` already finished landing (successfully or
399/// not) - that path already carries its own record of what happened - and
400/// `due` gives an operator who is about to fix a run by hand a grace window
401/// before the janitor starts asking the same question about it.
402pub fn eligible_for_external_merge_check(
403    status: RunStatus,
404    merge_recorded: bool,
405    updated_at: Timestamp,
406    now: Timestamp,
407    grace_secs: u64,
408) -> bool {
409    status == RunStatus::Blocked && !merge_recorded && due(now, updated_at, grace_secs)
410}
411
412/// Rotate `ids` so a fixed per-pass cap does not always land on the same
413/// prefix of the list, starting at `start` (already reduced modulo nothing -
414/// callers pass whatever a persisted cursor last recorded, and this takes it
415/// modulo `ids.len()` itself).
416///
417/// Pure: the offset is the caller's problem, not wall-clock time. An earlier
418/// version derived `start` from `now.as_second() % ids.len()`, which looked
419/// independent of any state to persist but is not actually independent of
420/// how often the janitor runs - a poll interval and a pass's own duration
421/// that alias onto the same handful of `now.as_second()` values (say, a
422/// ten-second poll and a pass that finishes in under a second) revisit only
423/// those same few residues forever and never advance into the rest of the
424/// list at all. [`reconcile_external_merges`] instead persists a cursor
425/// across passes ([`read_external_merge_cursor`] /
426/// [`write_external_merge_cursor`]) and advances it by exactly how many ids
427/// this pass actually looked at, which is the only quantity that is
428/// guaranteed to move forward pass over pass regardless of timing.
429fn rotate_from(ids: &[String], start: usize) -> Vec<String> {
430    if ids.is_empty() {
431        return Vec::new();
432    }
433    let start = start % ids.len();
434    ids[start..]
435        .iter()
436        .chain(ids[..start].iter())
437        .cloned()
438        .collect()
439}
440
441/// Where [`reconcile_external_merges`] remembers how far it got, so a fixed
442/// per-pass cap advances through a fleet's eligible runs pass over pass
443/// instead of camping on whichever ones happen to sort first.
444const EXTERNAL_MERGE_CURSOR_FILE: &str = "external-merge-cursor";
445
446/// `0` for anything this cannot read as a plain number - a first run, a
447/// corrupt or missing file, a build that has never written one - which is
448/// exactly as good a starting point as any: the cursor's whole job is to
449/// keep moving, not to encode any particular position as meaningful.
450fn read_external_merge_cursor(home: &Path) -> usize {
451    std::fs::read_to_string(home.join(EXTERNAL_MERGE_CURSOR_FILE))
452        .ok()
453        .and_then(|s| s.trim().parse().ok())
454        .unwrap_or(0)
455}
456
457/// Best-effort, like every other write in this module's sweep: a failure to
458/// persist the cursor costs the fleet one pass's worth of forward progress
459/// on this sweep, never the run whose housekeeping it is racing to keep up
460/// with.
461fn write_external_merge_cursor(home: &Path, cursor: usize) {
462    if let Err(e) = std::fs::write(home.join(EXTERNAL_MERGE_CURSOR_FILE), cursor.to_string()) {
463        tracing::warn!("housekeep: persist the external-merge sweep's cursor: {e:#}");
464    }
465}
466
467/// Close the gap `magi fold --merged` exists for, without waiting for an
468/// operator to notice and go find the pull request URL: a run that stopped
469/// `Blocked` with no `merge` recorded may have been merged anyway, by hand,
470/// on a pull request `land::land` never opened or never got to observe as
471/// merged. `land::find_external_merge` asks GitHub about the run's own
472/// winning branch, and only a pull request it can uniquely tie back to this
473/// run is ever acted on - see that function's own doc for what "uniquely"
474/// excludes.
475///
476/// A run this finds and fixes goes through
477/// [`crate::land::correct_confirmed_external_merge`] - the same correction
478/// `magi fold --merged` performs, minus the same-repo guard that command
479/// needs and this loop doesn't (see that function's own doc: the URL here
480/// was never operator-supplied, it came from asking `state.repo`'s own
481/// remote) - then through [`crate::graph::fold_run`] so it stops holding
482/// worktrees the moment it stops needing them. A run this cannot decide
483/// about - `gh` unreachable, more than one candidate pull
484/// request, nothing found at all - is left exactly as it is; only an error
485/// asking GitHub raises a notice, since "nothing found" is the ordinary,
486/// expected shape of a run that really is just blocked.
487async fn reconcile_external_merges(runs: &Path, home: &Path, disk: &Disk, now: Timestamp) -> usize {
488    reconcile_external_merges_with(runs, home, disk, now, |state| async move {
489        crate::land::find_external_merge(&state).await
490    })
491    .await
492}
493
494async fn reconcile_external_merges_with<F, Fut>(
495    runs: &Path,
496    home: &Path,
497    disk: &Disk,
498    now: Timestamp,
499    check: F,
500) -> usize
501where
502    F: Fn(RunState) -> Fut,
503    Fut: std::future::Future<Output = Result<Option<crate::land::ExternalMerge>>>,
504{
505    let mut ids: Vec<String> = std::fs::read_dir(runs)
506        .into_iter()
507        .flatten()
508        .flatten()
509        .filter(|e| e.path().join("run.json").is_file())
510        .map(|e| e.file_name().to_string_lossy().into_owned())
511        .collect();
512    ids.sort_unstable();
513
514    // Every eligible id first, cheaply (a `Meta` read, no `gh`), then rotated
515    // before the cap is applied. Capping the sorted order directly would
516    // always land on the same lexicographic prefix - a fleet's oldest run ids
517    // - so a handful of long-blocked runs that never turn out to be merged
518    // would starve every id after them of a `gh` check, forever.
519    let mut eligible: Vec<String> = Vec::new();
520    for id in ids {
521        if crate::daemon::is_working_on(home, &id, now) {
522            continue;
523        }
524        let meta = match read_external_merge_meta(runs, &id) {
525            Ok(meta) => meta,
526            // Already counted as unreadable by `fold_due`'s own pass; not
527            // worth a second warning for the same file.
528            Err(_) => continue,
529        };
530        if eligible_for_external_merge_check(
531            meta.status,
532            meta.merge.is_some(),
533            meta.updated_at,
534            now,
535            disk.fold_grace_secs,
536        ) {
537            eligible.push(id);
538        }
539    }
540
541    if eligible.is_empty() {
542        return 0;
543    }
544    let cursor = read_external_merge_cursor(home);
545    let window: Vec<String> = rotate_from(&eligible, cursor)
546        .into_iter()
547        .take(MAX_EXTERNAL_MERGE_CHECKS_PER_PASS)
548        .collect();
549    // Advances by exactly how many ids this pass looked at, regardless of
550    // what came of looking - a run that turned out not to be merged after
551    // all must not be revisited before every other eligible run has had its
552    // own turn.
553    write_external_merge_cursor(home, (cursor + window.len()) % eligible.len());
554
555    let mut reconciled = 0usize;
556    for id in window {
557        let mut state = match read_state(runs, &id) {
558            Ok(state) => state,
559            Err(_) => continue,
560        };
561        if let Ok(record) = serde_json::to_value(&state)
562            && let Some(reason) =
563                crate::external_merge_attention::historical_reason(home, &id, &record)
564        {
565            tracing::info!("housekeep: external merge for {id}: {reason}");
566            continue;
567        }
568        match check(state.clone()).await {
569            Ok(Some(found)) => {
570                match crate::land::correct_confirmed_external_merge(&mut state, &found.url).await {
571                    Ok(_) => {
572                        if let Err(e) = crate::graph::fold_run(&mut state, true, home).await {
573                            tracing::warn!(
574                                "housekeep: fold {id} after recording its external merge: {e:#}"
575                            );
576                        }
577                        reconciled += 1;
578                    }
579                    Err(e) => tracing::warn!(
580                        "housekeep: record external merge {} for {id}: {e:#}",
581                        found.url
582                    ),
583                }
584            }
585            Ok(None) => {}
586            Err(e) => {
587                tracing::warn!("housekeep: check external merge for {id}: {e:#}");
588                crate::notices::raise_in(
589                    home,
590                    crate::notices::Notice::warn(
591                        &format!("merged-unrecorded:{id}"),
592                        format!(
593                            "Run {id} is blocked with no recorded merge, and checking GitHub \
594                             for a merge failed; check by hand."
595                        ),
596                    )
597                    .link(crate::notices::Link::Run { id: id.clone() }),
598                );
599            }
600        }
601    }
602    reconciled
603}
604
605/// Read a whole run state from a runs directory, for folding only.
606///
607/// Deliberately more permissive than [`RunState::load`], which this does not
608/// call: `load` backs `--resume` and every hand-driven command, where a
609/// schema this build disagrees with the *meaning* of must refuse outright
610/// rather than resume a review round or a tally against stale semantics
611/// (`RunState::SCHEMA`'s own docs list what has changed meaning at each
612/// bump). Folding recomputes nothing - it only reads worktree paths, branch
613/// names and a tally winner off the struct to remove them - so an old
614/// schema's values are exactly as good here as a current one's; every schema
615/// bump so far has only ever added a field or a variant, never repurposed an
616/// existing one, and serde already fills an added field's default when an
617/// older record has nothing to say about it. What this cannot tolerate, and
618/// what still surfaces as an `Err`, is `run.json` failing to parse at all.
619fn read_state(runs: &Path, id: &str) -> Result<RunState> {
620    let path = runs.join(id).join("run.json");
621    let body =
622        std::fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
623    let state: RunState =
624        serde_json::from_str(&body).with_context(|| format!("parse {}", path.display()))?;
625    Ok(state)
626}
627
628/// Remove only verified clean Git worktrees and empty directory scaffolding.
629/// Unknown files or unreadable Git state require recovery, never recursive deletion.
630async fn remove_clean_bay(root: &Path) -> Result<()> {
631    let mut pending = vec![root.to_path_buf()];
632    let mut directories = Vec::new();
633    let mut worktrees = Vec::new();
634    while let Some(path) = pending.pop() {
635        if path.join(".git").exists() {
636            if !crate::git::is_clean(&path).await? {
637                anyhow::bail!("uncommitted work in {}; recovery required", path.display());
638            }
639            worktrees.push(path);
640        } else {
641            for entry in std::fs::read_dir(&path)? {
642                let entry = entry?;
643                if !entry.file_type()?.is_dir() {
644                    anyhow::bail!(
645                        "unknown worktree content in {}; recovery required",
646                        path.display()
647                    );
648                }
649                pending.push(entry.path());
650            }
651            directories.push(path);
652        }
653    }
654    for path in worktrees {
655        if !crate::git::worktree_remove_clean(&path, &path).await? {
656            anyhow::bail!("clean worktree removal refused: {}", path.display());
657        }
658    }
659    for path in directories.into_iter().rev() {
660        std::fs::remove_dir(path)?;
661    }
662    Ok(())
663}
664
665/// Remove a run that cannot be read: its state directory under `runs` and its
666/// worktree directory under `worktrees_root`.
667///
668/// Inspect worktrees before deleting any state or artifacts. Only clean registered
669/// worktrees and empty directory scaffolding can be reclaimed; unknown or dirty
670/// content remains available for recovery. Nonstandard worktree roots remain the
671/// operator's responsibility when the state file cannot identify them.
672pub async fn fold_unreadable(runs: &Path, worktrees_root: &Path, id: &str) -> Result<Vec<String>> {
673    let resolved = resolve_id_path(runs, id)?;
674    let mut removed = Vec::new();
675    let run_dir = runs.join(&resolved);
676    let wt = worktrees_root.join(short_of(&resolved));
677    if wt.exists() {
678        remove_clean_bay(&wt).await?;
679        removed.push(wt.to_string_lossy().into_owned());
680    }
681    // Keep the recovery artifacts if any worktree could not be removed safely.
682    if run_dir.exists() {
683        std::fs::remove_dir_all(&run_dir)
684            .with_context(|| format!("remove {}", run_dir.display()))?;
685        removed.push(format!("runs/{resolved}"));
686    }
687    Ok(removed)
688}
689
690/// Reclaim worktrees under `worktrees_root` that no run record in `runs`
691/// claims anymore, and return how many were removed.
692///
693/// [`fold_due`] only ever sees a worktree by walking `runs/` first, so a
694/// worktree whose run record is already gone — `magi run rm`, or a record
695/// deleted before its worktree — never enters that loop at all: nothing there
696/// is looking for it. This walks the worktree bay directly instead, and
697/// removes any `<short>` directory that no run id maps to.
698///
699/// Two things must never happen, and this checks both before ever touching a
700/// directory:
701///
702/// - **A worktree bay is never the only kind of thing under `worktrees_root`,
703///   and this must not assume it is.** A hand-placed scratch directory, or
704///   anything else an operator or another tool left in the same bay, has the
705///   same "no run claims it" shape as a genuine orphan but is not one -
706///   [`looks_like_a_worktree_bay`] is the same tag shape [`crate::run::is_run_id`]
707///   already requires of a real run's short id, and anything else is left
708///   alone regardless of what else is true about it.
709/// - **A worktree that was only just created might not have a `run.json` yet
710///   for a reason that has nothing to do with being orphaned.** `Runner::start`
711///   and `Runner::review` both create the worktree before the first
712///   `RunState::save` lands, and that gap - several `git` subprocesses wide -
713///   is invisible to [`crate::daemon::is_working_on_short`] whenever the run
714///   is not being driven through this daemon's own `poll` loop at all (a
715///   `magi review` invocation, for one). A directory whose own modification
716///   time is within `grace_secs` of `now` is left alone on that basis alone,
717///   the same margin [`fold_due`] gives a run before treating it as truly
718///   finished - long enough that no realistic gap between a `worktree add`
719///   and its `run.json` could ever be mistaken for one.
720///
721/// The one failure this must never cause is deleting the worktree of a run
722/// that is genuinely in flight. [`crate::daemon::is_working_on_short`] is the
723/// same liveness check [`fold_due`] trusts everywhere else in this module,
724/// checked by short id because there is no full id to compare here; when it
725/// cannot tell, this leaves the directory alone. Best-effort like the rest of
726/// housekeeping: one directory git or the filesystem refuses to give up is a
727/// `tracing::warn`, not a reason to abandon the rest of the pass.
728pub async fn fold_orphaned_worktrees(
729    runs: &Path,
730    worktrees_root: &Path,
731    home: &Path,
732    grace_secs: u64,
733    now: Timestamp,
734) -> usize {
735    let known: std::collections::HashSet<String> = std::fs::read_dir(runs)
736        .into_iter()
737        .flatten()
738        .flatten()
739        .map(|e| e.file_name().to_string_lossy().into_owned())
740        .filter(|name| crate::run::is_run_id(name))
741        .map(|id| short_of(&id).to_owned())
742        .collect();
743
744    let mut folded = 0usize;
745    for entry in std::fs::read_dir(worktrees_root)
746        .into_iter()
747        .flatten()
748        .flatten()
749    {
750        if !entry.path().is_dir() {
751            continue;
752        }
753        let short = entry.file_name().to_string_lossy().into_owned();
754        if !looks_like_a_worktree_bay(&short) {
755            continue;
756        }
757        if known.contains(&short) || crate::daemon::is_working_on_short(home, &short, now) {
758            continue;
759        }
760        let wt = entry.path();
761        if !stale_enough(&wt, grace_secs, now) {
762            continue;
763        }
764        match remove_clean_bay(&wt).await {
765            Ok(()) => folded += 1,
766            Err(e) => tracing::warn!(
767                "housekeep: remove orphaned worktree {}: {e:#}",
768                wt.display()
769            ),
770        }
771    }
772    folded
773}
774
775/// Does `name` have the shape a run's own worktree bay is named with: the
776/// same 4-character alphanumeric tag [`crate::run::is_run_id`] requires of a
777/// full id's trailing block (see [`short_of`])?
778///
779/// Anything else under `worktrees_root` is not a bay this function reclaims
780/// at all, claimed or not - answering "does a run claim this?" about a
781/// directory that was never a run's worktree in the first place is exactly
782/// the wrong question to ask before deleting it.
783fn looks_like_a_worktree_bay(name: &str) -> bool {
784    name.len() == 4 && name.bytes().all(|b| b.is_ascii_alphanumeric())
785}
786
787/// Is `dir`'s own modification time old enough, against `grace_secs` and
788/// `now`, that its emptiness of a run record can be trusted rather than
789/// caught mid-creation?
790///
791/// A directory this pass cannot stat at all - a race with its own removal, a
792/// permission error - is treated as not yet stale: unreadable metadata is not
793/// evidence of anything, and the janitor already keeps rather than deletes
794/// whenever it cannot tell (see the module docs).
795///
796/// `grace_secs` is floored at [`MIN_ORPHAN_AGE_SECS`] regardless of what the
797/// caller passes: `0` is a documented, legitimate value for
798/// [`crate::config::Disk::fold_grace_secs`] (`due`'s own "always due" case),
799/// because that grace answers a policy question the operator owns - how long
800/// a *known, finished* run's worktree lingers before cleanup. Whether an
801/// orphan worktree is actually a race with `Runner::review`'s `git worktree
802/// add` landing before its `run.json` is not a policy question, and must not
803/// collapse to zero just because the operator turned the other grace off -
804/// that would defeat the very check meant to catch it.
805fn stale_enough(dir: &Path, grace_secs: u64, now: Timestamp) -> bool {
806    let Ok(modified) = std::fs::metadata(dir).and_then(|m| m.modified()) else {
807        return false;
808    };
809    let Ok(ts) = Timestamp::try_from(modified) else {
810        return false;
811    };
812    due(now, ts, grace_secs.max(MIN_ORPHAN_AGE_SECS))
813}
814
815/// The floor under [`stale_enough`]'s grace, independent of
816/// [`crate::config::Disk::fold_grace_secs`].
817///
818/// Ample next to the race it guards: the gap between `Runner::start` or
819/// `Runner::review` creating a worktree and the first `RunState::save`
820/// landing is a handful of `git` subprocess calls, not minutes - but the
821/// janitor cannot tell "still mid-setup" from "orphaned" by any other signal
822/// for a run that never registers with `daemon::Status` at all (a `magi
823/// review` invocation, for one), so this is generous on purpose rather than
824/// tuned to the observed case.
825const MIN_ORPHAN_AGE_SECS: u64 = 5 * 60;
826
827/// Resolve an id or prefix against an explicit runs directory, exactly the way
828/// [`crate::run::resolve_id`] does against the global home.
829fn resolve_id_path(runs: &Path, prefix: &str) -> Result<String> {
830    // Keyed on the directory, not on a readable state file: the record this
831    // route exists to remove may be a lone `run.json.tmp` from a save that
832    // ran out of disk, and that is precisely the one a human needs a way to
833    // clear (see `crate::run::list_ids`).
834    if runs.join(prefix).is_dir() && crate::run::is_run_id(prefix) {
835        return Ok(prefix.to_owned());
836    }
837    let mut hits: Vec<String> = Vec::new();
838    for e in std::fs::read_dir(runs).into_iter().flatten().flatten() {
839        if !e.path().is_dir() {
840            continue;
841        }
842        let id = e.file_name().to_string_lossy().into_owned();
843        if crate::run::is_run_id(&id) && (id.starts_with(prefix) || id.ends_with(prefix)) {
844            hits.push(id);
845        }
846    }
847    match hits.len() {
848        1 => Ok(hits.into_iter().next().expect("exactly one hit")),
849        0 => bail!("no run matches `{prefix}`"),
850        _ => bail!(
851            "`{prefix}` matches {} runs: {}",
852            hits.len(),
853            hits.join(", ")
854        ),
855    }
856}
857
858/// `magi fold`'s recovery path for a run whose worktrees are already gone —
859/// so [`crate::graph::fold_run`] removed nothing — but whose `run.json` still
860/// lists active seats nobody is left to answer for: no live daemon claims the
861/// run, and every one of those seats has overrun its own timeout (see
862/// [`RunState::active_all_overrun`]). Clearing them and failing the run is
863/// what lets it be deleted afterward — [`RunState::ensure_can_delete`] only
864/// ever checks whether a live daemon is working on the run and whether its
865/// candidates are folded, not `status`, but a run stuck `implementing`
866/// forever with an empty worktree still reads as unresolved everywhere else
867/// (`magi show`, the deck, the phone) until this runs.
868///
869/// Returns `false` without changing anything when a live daemon still claims
870/// the run, or when some active seat has not actually overrun its budget yet
871/// — a run that is merely between waves must never be guessed at.
872pub fn clear_abandoned_active(state: &mut RunState, home: &Path, now: Timestamp) -> Result<bool> {
873    if crate::daemon::is_working_on(home, &state.id, now) || !state.active_all_overrun(now) {
874        return Ok(false);
875    }
876    state.abandon("fold");
877    state.save_under(home)?;
878    // The seat that asked is gone for good now - the same door
879    // `graph::Runner::settle_questions` closes the moment `status` lands
880    // somewhere non-resumable, see that method's own doc. Without this, an
881    // open question the abandoned seat left behind would keep badging the
882    // operator until the next daemon startup's `abandon_settled_questions`
883    // pass happened to notice it, or forever if nothing is running `magi
884    // serve` at all.
885    if let Err(e) = Questions::at(home.join("questions")).settle_run(&state.id, state.status) {
886        tracing::warn!("abandon questions for {}: {e:#}", state.id);
887    }
888    Ok(true)
889}
890
891/// Delete files from the shared build cache until it fits its cap — but only
892/// while nobody live is registered as using it. [`crate::cache::maintenance_prune`]
893/// takes out the same lease a build would, so a prune can never race a
894/// compile in flight (this run's own, another run's, or a human's `magi
895/// review`) into deleting a file that build still needs. `Ok(None)` when the
896/// cache is in use right now; the next pass catches it once the borrower
897/// releases it, the same way a cap of `0` or a missing `CARGO_TARGET_DIR`
898/// already meant "nothing to do this time" here.
899pub fn prune_cache(home: &Path, cache: &Path, limit_bytes: u64) -> Result<Option<Prune>> {
900    crate::cache::maintenance_prune(home, cache, limit_bytes)
901}
902
903/// [`prune_cache`], but resolving the operator's opt-out and missing
904/// `CARGO_TARGET_DIR` first — the same two checks [`housekeep`]'s idle pass
905/// makes before ever measuring the cache, factored out so
906/// [`crate::daemon`]'s between-runs check (see the module's own doc for why
907/// congestion can make "idle" arrive too rarely to matter) makes them
908/// identically rather than growing its own copy that could drift. `Ok(None)`
909/// covers a cap of `0` (see the module docs on `cache_limit_bytes`), a config
910/// that renders no `CARGO_TARGET_DIR` to aggregate at all, and a cache
911/// currently in use (see [`prune_cache`]).
912pub fn prune_cache_if_over_limit(
913    cfg: &crate::config::Config,
914    home: &Path,
915) -> Result<Option<Prune>> {
916    if cfg.disk.cache_limit_bytes == 0 {
917        return Ok(None);
918    }
919    let Some(cache) = cfg.cache_dir() else {
920        return Ok(None);
921    };
922    let limit = crate::disk::free_bytes(&cache)
923        .ok()
924        .map_or(cfg.disk.cache_limit_bytes, |free| {
925            crate::disk::pressure_limit(
926                dir_size(&cache),
927                cfg.disk.cache_limit_bytes,
928                cfg.disk.min_free_bytes,
929                free,
930            )
931        });
932    prune_cache(home, &cache, limit)
933}
934
935/// The cache's path, size and cap, for `magi cache show` and the health view.
936/// `None` when the config declares no `CARGO_TARGET_DIR` to aggregate.
937///
938/// A cap of `0` means the operator opted out of pruning; the size is then
939/// reported but never acted on.
940pub fn cache_report(cfg: &crate::config::Config) -> Option<(PathBuf, u64, u64)> {
941    let cache = cfg.cache_dir()?;
942    Some((
943        cache.clone(),
944        cache_size(&cache),
945        cfg.disk.cache_limit_bytes,
946    ))
947}
948
949/// Size in bytes of the shared build cache.
950pub fn cache_size(cache: &Path) -> u64 {
951    dir_size(cache)
952}
953
954#[cfg(test)]
955mod tests {
956
957    async fn sh(cwd: &Path, args: &[&str]) {
958        let out = crate::git::git_raw(cwd, args).await.expect("spawn git");
959        assert!(out.ok(), "git {args:?}: {}", out.stderr);
960    }
961
962    async fn commit(cwd: &Path, file: &str, body: &str) {
963        std::fs::write(cwd.join(file), body).unwrap();
964        sh(cwd, &["add", file]).await;
965        sh(
966            cwd,
967            &[
968                "-c",
969                "user.name=t",
970                "-c",
971                "user.email=t@example.com",
972                "commit",
973                "-q",
974                "-m",
975                body,
976            ],
977        )
978        .await;
979    }
980
981    const LONG: std::time::Duration = std::time::Duration::from_secs(30);
982
983    #[tokio::test]
984    async fn fetch_advances_origin_main_and_leaves_the_checkout_alone() {
985        let tmp = tempfile::tempdir().unwrap();
986        let t = tmp.path();
987        let bare = t.join("remote.git");
988        sh(
989            t,
990            &["init", "-q", "--bare", "-b", "main", bare.to_str().unwrap()],
991        )
992        .await;
993        let dir = t.join("root/h/o/r");
994        std::fs::create_dir_all(dir.parent().unwrap()).unwrap();
995        sh(
996            t,
997            &["clone", "-q", bare.to_str().unwrap(), dir.to_str().unwrap()],
998        )
999        .await;
1000        sh(&dir, &["checkout", "-q", "-b", "main"]).await;
1001        commit(&dir, "a.txt", "one").await;
1002        sh(&dir, &["push", "-q", "origin", "main"]).await;
1003        sh(&dir, &["checkout", "-q", "--detach"]).await;
1004        std::fs::write(dir.join("a.txt"), "dirty").unwrap();
1005
1006        let other = t.join("other");
1007        sh(
1008            t,
1009            &[
1010                "clone",
1011                "-q",
1012                bare.to_str().unwrap(),
1013                other.to_str().unwrap(),
1014            ],
1015        )
1016        .await;
1017        commit(&other, "b.txt", "two").await;
1018        sh(&other, &["push", "-q", "origin", "HEAD:main"]).await;
1019
1020        let rev = |d: PathBuf, r: &'static str| async move {
1021            crate::git::git(&d, &["rev-parse", r]).await.unwrap()
1022        };
1023        let head = rev(dir.clone(), "HEAD").await;
1024        let before = rev(dir.clone(), "origin/main").await;
1025        let status = crate::git::git(&dir, &["status", "--porcelain"])
1026            .await
1027            .unwrap();
1028
1029        let r = fetch_origins(&[t.join("root")], LONG, || false).await;
1030        assert_eq!(
1031            r,
1032            FetchReport {
1033                fetched: 1,
1034                no_origin: 0,
1035                failed: 0
1036            }
1037        );
1038
1039        assert_ne!(rev(dir.clone(), "origin/main").await, before);
1040        assert_eq!(rev(dir.clone(), "HEAD").await, head);
1041        assert_eq!(
1042            crate::git::git(&dir, &["status", "--porcelain"])
1043                .await
1044                .unwrap(),
1045            status
1046        );
1047        assert_eq!(std::fs::read_to_string(dir.join("a.txt")).unwrap(), "dirty");
1048    }
1049
1050    #[tokio::test]
1051    async fn fetch_never_writes_a_local_branch_whatever_the_configured_refspec() {
1052        let tmp = tempfile::tempdir().unwrap();
1053        let t = tmp.path();
1054        let bare = t.join("remote.git");
1055        sh(
1056            t,
1057            &["init", "-q", "--bare", "-b", "main", bare.to_str().unwrap()],
1058        )
1059        .await;
1060        let dir = t.join("root/h/o/r");
1061        std::fs::create_dir_all(dir.parent().unwrap()).unwrap();
1062        sh(
1063            t,
1064            &["clone", "-q", bare.to_str().unwrap(), dir.to_str().unwrap()],
1065        )
1066        .await;
1067        sh(&dir, &["checkout", "-q", "-b", "main"]).await;
1068        commit(&dir, "a.txt", "one").await;
1069        sh(&dir, &["push", "-q", "origin", "main"]).await;
1070        sh(&dir, &["checkout", "-q", "--detach"]).await;
1071        // A hostile mapping onto the local branch, and no origin/* at all.
1072        sh(
1073            &dir,
1074            &[
1075                "config",
1076                "remote.origin.fetch",
1077                "+refs/heads/main:refs/heads/main",
1078            ],
1079        )
1080        .await;
1081        let _ = crate::git::git_raw(&dir, &["update-ref", "-d", "refs/remotes/origin/main"]).await;
1082
1083        let other = t.join("other");
1084        sh(
1085            t,
1086            &[
1087                "clone",
1088                "-q",
1089                bare.to_str().unwrap(),
1090                other.to_str().unwrap(),
1091            ],
1092        )
1093        .await;
1094        commit(&other, "b.txt", "two").await;
1095        sh(&other, &["push", "-q", "origin", "HEAD:main"]).await;
1096        let remote_tip = crate::git::git(&other, &["rev-parse", "HEAD"])
1097            .await
1098            .unwrap();
1099        let local_main = crate::git::git(&dir, &["rev-parse", "refs/heads/main"])
1100            .await
1101            .unwrap();
1102
1103        let r = fetch_origins(&[t.join("root")], LONG, || false).await;
1104        assert_eq!(r.fetched, 1);
1105        assert_eq!(
1106            crate::git::git(&dir, &["rev-parse", "refs/remotes/origin/main"])
1107                .await
1108                .unwrap(),
1109            remote_tip
1110        );
1111        assert_eq!(
1112            crate::git::git(&dir, &["rev-parse", "refs/heads/main"])
1113                .await
1114                .unwrap(),
1115            local_main
1116        );
1117    }
1118
1119    #[tokio::test]
1120    async fn fetch_skips_no_origin_and_survives_an_unreachable_origin() {
1121        let tmp = tempfile::tempdir().unwrap();
1122        let t = tmp.path();
1123        let root = t.join("root");
1124        let bare = t.join("remote.git");
1125        sh(t, &["init", "-q", "--bare", bare.to_str().unwrap()]).await;
1126
1127        let none = root.join("h/o/none");
1128        let dead = root.join("h/o/dead");
1129        let good = root.join("h/o/good");
1130        for d in [&none, &dead, &good] {
1131            std::fs::create_dir_all(d).unwrap();
1132            sh(d, &["init", "-q"]).await;
1133        }
1134        sh(
1135            &dead,
1136            &[
1137                "remote",
1138                "add",
1139                "origin",
1140                t.join("missing").to_str().unwrap(),
1141            ],
1142        )
1143        .await;
1144        sh(&good, &["remote", "add", "origin", bare.to_str().unwrap()]).await;
1145
1146        let r = fetch_origins(&[root], LONG, || false).await;
1147        assert_eq!(
1148            r,
1149            FetchReport {
1150                fetched: 1,
1151                no_origin: 1,
1152                failed: 1
1153            }
1154        );
1155    }
1156    use super::*;
1157    use crate::config::Disk;
1158    use std::fs;
1159
1160    fn ts(s: &str) -> Timestamp {
1161        s.parse().expect("rfc3339")
1162    }
1163
1164    fn block_on<F: std::future::Future>(f: F) -> F::Output {
1165        tokio::runtime::Runtime::new().expect("runtime").block_on(f)
1166    }
1167
1168    #[test]
1169    fn a_run_is_due_after_its_grace_and_not_before() {
1170        let now = ts("2026-09-05T00:00:00Z");
1171        let grace = 600;
1172        let old = now - SignedDuration::new(601, 0);
1173        let fresh = now - SignedDuration::new(599, 0);
1174        assert!(due(now, old, grace));
1175        assert!(!due(now, fresh, grace));
1176        // Exactly at the edge: not yet due.
1177        let edge = now - SignedDuration::new(600, 0);
1178        assert!(!due(now, edge, grace));
1179        // A zero grace folds everything, ever.
1180        assert!(due(now, old, 0));
1181    }
1182
1183    #[test]
1184    fn rotate_from_moves_the_starting_point_as_the_cursor_advances() {
1185        let ids: Vec<String> = ["a", "b", "c", "d", "e"]
1186            .iter()
1187            .map(|s| s.to_string())
1188            .collect();
1189
1190        // A cursor of 0 starts at the front, same as an unrotated list.
1191        assert_eq!(rotate_from(&ids, 0), ids);
1192
1193        // A cursor of 2 starts two ids further along, with the skipped
1194        // front wrapping to the tail rather than being dropped.
1195        let at_2 = rotate_from(&ids, 2);
1196        assert_eq!(at_2, vec!["c", "d", "e", "a", "b"]);
1197        for id in &ids {
1198            assert!(at_2.contains(id));
1199        }
1200
1201        // A cursor past the list's own length wraps rather than panicking -
1202        // the caller never has to reduce it modulo anything itself.
1203        assert_eq!(rotate_from(&ids, 7), rotate_from(&ids, 2));
1204
1205        // An empty list has no start point to compute and must not panic.
1206        assert_eq!(rotate_from(&[], 0), Vec::<String>::new());
1207    }
1208
1209    /// A fleet with more eligible runs than one pass can check must not park
1210    /// the same handful at the front of the window forever: advancing the
1211    /// cursor by exactly how many ids one pass actually looked at - the
1212    /// arithmetic [`reconcile_external_merges`] itself does - walks every id
1213    /// to the front within one full rotation, with no reliance on how often
1214    /// or how regularly passes happen to run.
1215    #[test]
1216    fn advancing_the_cursor_by_the_window_size_gives_every_id_a_turn() {
1217        let ids: Vec<String> = (0..12).map(|n| format!("run-{n}")).collect();
1218        let cap = 5usize;
1219        let mut cursor = 0usize;
1220        let mut ever_seen: std::collections::HashSet<String> = std::collections::HashSet::new();
1221        for _ in 0..ids.len() {
1222            let window: Vec<String> = rotate_from(&ids, cursor).into_iter().take(cap).collect();
1223            ever_seen.extend(window.iter().cloned());
1224            cursor = (cursor + window.len()) % ids.len();
1225        }
1226        assert_eq!(
1227            ever_seen.len(),
1228            ids.len(),
1229            "every id must be checked at least once across a full rotation, whatever \
1230             the cadence between passes"
1231        );
1232    }
1233
1234    #[test]
1235    fn the_external_merge_cursor_round_trips_through_a_files_absence() {
1236        let dir = tempfile::tempdir().unwrap();
1237        let home = dir.path();
1238
1239        // Nothing written yet: starts at 0, not an error.
1240        assert_eq!(read_external_merge_cursor(home), 0);
1241
1242        write_external_merge_cursor(home, 7);
1243        assert_eq!(read_external_merge_cursor(home), 7);
1244
1245        // Anything unreadable as a plain number falls back to 0 rather than
1246        // wedging the sweep on a corrupt file.
1247        std::fs::write(home.join(EXTERNAL_MERGE_CURSOR_FILE), "not a number").unwrap();
1248        assert_eq!(read_external_merge_cursor(home), 0);
1249    }
1250
1251    #[test]
1252    fn a_run_qualifies_for_an_external_merge_check_only_when_blocked_unmerged_and_due() {
1253        let now = ts("2026-09-05T00:00:00Z");
1254        let grace = 600;
1255        let old = now - SignedDuration::new(601, 0);
1256        let fresh = now - SignedDuration::new(599, 0);
1257
1258        assert!(
1259            eligible_for_external_merge_check(RunStatus::Blocked, false, old, now, grace),
1260            "blocked, unmerged, and past its grace period is exactly the run this exists for"
1261        );
1262        assert!(
1263            !eligible_for_external_merge_check(RunStatus::Blocked, false, fresh, now, grace),
1264            "an operator mid-fix deserves the same grace window `fold_due` gives before \
1265             the janitor starts asking GitHub about it"
1266        );
1267        assert!(
1268            !eligible_for_external_merge_check(RunStatus::Blocked, true, old, now, grace),
1269            "a run `land::land` already recorded a merge outcome for has its own answer \
1270             already; this check is only for a run with nothing recorded at all"
1271        );
1272        assert!(
1273            !eligible_for_external_merge_check(RunStatus::Stalled, false, old, now, grace),
1274            "stalled is not blocked - it means the tally never reached quorum, which a \
1275             pull request cannot fix"
1276        );
1277        assert!(
1278            !eligible_for_external_merge_check(RunStatus::Ready, false, old, now, grace),
1279            "ready has nothing to correct - it was never landed by design"
1280        );
1281        assert!(
1282            !eligible_for_external_merge_check(RunStatus::Superseded, false, old, now, grace),
1283            "a superseded run's task already has its answer from a later attempt; \
1284             there is nothing left for GitHub to confirm here"
1285        );
1286    }
1287
1288    /// A blocked run with a winner in an isolated repository directory.
1289    /// Consumer tests inject forge results without invoking a real CLI.
1290    fn write_blocked_run(runs: &Path, id: &str, updated_at: Timestamp) {
1291        use crate::run::{Candidate, Tally};
1292        use std::collections::BTreeMap;
1293
1294        let mut state = RunState::new(
1295            runs.parent().unwrap().to_path_buf(),
1296            "main".to_owned(),
1297            "0000000000000000000000000000000000000000".to_owned(),
1298            String::new(),
1299            crate::config::Config::default(),
1300        );
1301        state.id = id.to_owned();
1302        state.status = RunStatus::Blocked;
1303        state.updated_at = updated_at;
1304        state.candidates.push(Candidate {
1305            index: 0,
1306            label: 'A',
1307            agent: "agent".to_owned(),
1308            branch: format!("magi/{id}/A"),
1309            worktree: runs.parent().unwrap().join("candidate"),
1310            summary: String::new(),
1311            stat: String::new(),
1312            files: 0,
1313            commits: 0,
1314            empty: false,
1315            failed: None,
1316            verified_noop: None,
1317            duration_ms: 0,
1318            folded: false,
1319        });
1320        state.tally = Some(Tally {
1321            first_choice: BTreeMap::from([('A', 1)]),
1322            borda: BTreeMap::new(),
1323            winner: 'A',
1324            rankings: 1,
1325            unanimous_initial: true,
1326            deliberated: false,
1327            changed_votes: 0,
1328            unanimous_final: true,
1329            tie_break: None,
1330            judges: 1,
1331            present: 1,
1332            quorum: 1,
1333            met_quorum: true,
1334            uncontested: None,
1335        });
1336        std::fs::create_dir_all(runs.join(id)).unwrap();
1337        std::fs::write(
1338            runs.join(id).join("run.json"),
1339            serde_json::to_string_pretty(&state).unwrap(),
1340        )
1341        .unwrap();
1342    }
1343
1344    /// No run here can be confirmed merged - one has a forge error,
1345    /// one has not sat blocked long enough, and one is
1346    /// claimed by a live daemon - so the sweep must leave every one of them
1347    /// exactly as it found them and never panic on the failures in between.
1348    #[tokio::test]
1349    async fn reconcile_external_merges_leaves_ineligible_and_unconfirmable_runs_alone() {
1350        let dir = tempfile::tempdir().unwrap();
1351        let runs = dir.path().join("runs");
1352        let home = dir.path().to_path_buf();
1353        std::fs::create_dir_all(&runs).unwrap();
1354
1355        let now = ts("2026-09-05T00:00:00Z");
1356        let disk = Disk {
1357            fold_grace_secs: 600,
1358            ..Disk::default()
1359        };
1360
1361        let due_id = "20260905-000000-blkd";
1362        write_blocked_run(&runs, due_id, ts("2026-08-01T00:00:00Z"));
1363
1364        let fresh_id = "20260905-000000-fres";
1365        write_blocked_run(&runs, fresh_id, now);
1366
1367        let live_id = "20260905-000000-live";
1368        write_blocked_run(&runs, live_id, ts("2026-08-01T00:00:00Z"));
1369        let mut status = crate::daemon::Status::new();
1370        status.current = vec![crate::daemon::Current {
1371            worker: None,
1372            task: "20260905-000000-task".to_owned(),
1373            run: live_id.to_owned(),
1374        }];
1375        status.updated_at = now;
1376        crate::daemon::write_status_to(&home.join("daemon.json"), &status).unwrap();
1377
1378        let reconciled = reconcile_external_merges_with(&runs, &home, &disk, now, |_| async {
1379            anyhow::bail!("injected forge authentication failure")
1380        })
1381        .await;
1382        assert_eq!(reconciled, 0, "a forge failure can never confirm a merge");
1383        for id in [due_id, fresh_id, live_id] {
1384            assert_eq!(
1385                read_meta(&runs, id).unwrap().status,
1386                RunStatus::Blocked,
1387                "{id} must be left exactly as it was found"
1388            );
1389        }
1390        let notices = crate::notices::Notices::at(home.join("notifications")).list();
1391        let due_notice = notices
1392            .iter()
1393            .find(|n| n.key == format!("merged-unrecorded:{due_id}"))
1394            .expect("notice for due_id must be raised");
1395        assert_eq!(
1396            due_notice.link,
1397            Some(crate::notices::Link::Run {
1398                id: due_id.to_owned(),
1399            })
1400        );
1401    }
1402
1403    #[tokio::test]
1404    async fn historical_external_merge_check_preserves_unknown_history() {
1405        let home = tempfile::tempdir().unwrap();
1406        let runs = home.path().join("runs");
1407        let id = "legacy";
1408        write_blocked_run(&runs, id, ts("2026-08-01T00:00:00Z"));
1409        let path = runs.join(id).join("run.json");
1410        let mut state = read_state(&runs, id).unwrap();
1411        state.repo = home.path().join("removed");
1412        state.candidates[0].folded = true;
1413        let original = serde_json::to_vec(&state).unwrap();
1414        std::fs::write(&path, &original).unwrap();
1415        let count = reconcile_external_merges_with(
1416            &runs,
1417            home.path(),
1418            &Disk::default(),
1419            ts("2026-11-14T00:00:00Z"),
1420            |_| async { panic!("historical unavailable checkout must not invoke the forge") },
1421        )
1422        .await;
1423        assert_eq!(count, 0);
1424        assert_eq!(std::fs::read(&path).unwrap(), original);
1425        assert!(
1426            crate::notices::Notices::at(home.path().join("notifications"))
1427                .all()
1428                .is_empty()
1429        );
1430
1431        // The same folded run is actionable when a current task owns it.
1432        let queue = crate::queue::Queue::at(home.path().join("queue"));
1433        let mut task = crate::queue::Task::new(
1434            "current".into(),
1435            "current".into(),
1436            state.repo.clone(),
1437            crate::queue::Source::Human,
1438        );
1439        task.runs.push(id.into());
1440        queue.put(&mut task).unwrap();
1441        reconcile_external_merges_with(
1442            &runs,
1443            home.path(),
1444            &Disk::default(),
1445            ts("2026-11-14T00:00:00Z"),
1446            |_| async { anyhow::bail!("current checkout unavailable") },
1447        )
1448        .await;
1449        assert_eq!(
1450            crate::notices::Notices::at(home.path().join("notifications"))
1451                .list()
1452                .len(),
1453            1
1454        );
1455        assert_eq!(std::fs::read(&path).unwrap(), original);
1456    }
1457
1458    /// The bug this exists to pin: a first version rotated on
1459    /// `now.as_second() % len`, which is fully determined by wall-clock time
1460    /// and nothing else. Two passes seconds apart - as they would be when a
1461    /// short poll interval and a fast pass alias onto the same handful of
1462    /// `now.as_second()` residues - landed on the *same* starting point and
1463    /// never covered a fleet bigger than the per-pass cap, however many
1464    /// times housekeeping ran. Persisting a cursor makes forward progress a
1465    /// function of how many ids got looked at, not of when the clock reads
1466    /// this pass happened to run.
1467    #[tokio::test]
1468    async fn reconcile_external_merges_covers_a_larger_fleet_across_repeated_passes_at_one_instant()
1469    {
1470        let dir = tempfile::tempdir().unwrap();
1471        let runs = dir.path().join("runs");
1472        let home = dir.path().to_path_buf();
1473        std::fs::create_dir_all(&runs).unwrap();
1474
1475        let now = ts("2026-09-05T00:00:00Z");
1476        let disk = Disk {
1477            fold_grace_secs: 600,
1478            ..Disk::default()
1479        };
1480        let old = ts("2026-08-01T00:00:00Z");
1481
1482        let ids: Vec<String> = (0..8).map(|n| format!("20260905-000000-r{n:03}")).collect();
1483        for id in &ids {
1484            write_blocked_run(&runs, id, old);
1485        }
1486
1487        let notices = crate::notices::Notices::at(home.join("notifications"));
1488
1489        // Two passes at the exact same `now`, exactly what a fast pass on a
1490        // short poll interval looks like. Each of the 8 injected forge-failure
1491        // runs raises its own notice the first time it is looked at, so the
1492        // count of distinct notices is a direct readout of how many distinct
1493        // ids have been checked so far.
1494        reconcile_external_merges_with(&runs, &home, &disk, now, |_| async {
1495            anyhow::bail!("injected forge authentication failure")
1496        })
1497        .await;
1498        let after_first = notices.list().len();
1499        assert_eq!(
1500            after_first, MAX_EXTERNAL_MERGE_CHECKS_PER_PASS,
1501            "the first pass checks exactly one cap's worth"
1502        );
1503
1504        reconcile_external_merges_with(&runs, &home, &disk, now, |_| async {
1505            anyhow::bail!("injected forge authentication failure")
1506        })
1507        .await;
1508        let after_second = notices.list().len();
1509        assert_eq!(
1510            after_second,
1511            ids.len(),
1512            "a second pass at the same instant must still reach every id the \
1513             first pass had no room for, not repeat the same cap's worth"
1514        );
1515    }
1516
1517    #[test]
1518    fn the_meta_reader_is_tolerant_of_everything_except_the_deciders() {
1519        let dir = tempfile::tempdir().unwrap();
1520        let runs = dir.path().join("runs");
1521        let id = "20260905-000000-abcd";
1522        std::fs::create_dir_all(runs.join(id)).unwrap();
1523        std::fs::write(
1524            runs.join(id).join("run.json"),
1525            r#"{"schema": 99, "id": "20260905-000000-abcd", "updated_at": "2026-09-05T00:00:00Z", "status": "ready", "junk_from_another_build": [1, 2, 3]}"#,
1526        )
1527        .unwrap();
1528        let meta = read_meta(&runs, id).expect("readable");
1529        assert_eq!(meta.status, RunStatus::Ready);
1530        assert_eq!(meta.updated_at, ts("2026-09-05T00:00:00Z"));
1531        assert!(read_meta(&runs, "nope").is_err(), "missing file unreadable");
1532        std::fs::write(runs.join(id).join("run.json"), "not json at all").unwrap();
1533        assert!(read_meta(&runs, id).is_err(), "garbage unreadable");
1534    }
1535
1536    #[test]
1537    fn fold_unreadable_preserves_unknown_content_and_recovery_artifacts() {
1538        let dir = tempfile::tempdir().unwrap();
1539        let runs = dir.path().join("runs");
1540        let wt = dir.path().join("wt");
1541        let id = "20260905-000000-abcd";
1542        std::fs::create_dir_all(runs.join(id).join("artifacts")).unwrap();
1543        std::fs::write(runs.join(id).join("run.json"), "garbage").unwrap();
1544        std::fs::create_dir_all(wt.join("abcd")).unwrap();
1545        std::fs::write(wt.join("abcd/leftover"), "owned edits").unwrap();
1546        assert!(block_on(fold_unreadable(&runs, &wt, id)).is_err());
1547        assert!(runs.join(id).join("artifacts").exists());
1548        assert_eq!(
1549            std::fs::read_to_string(wt.join("abcd/leftover")).unwrap(),
1550            "owned edits"
1551        );
1552    }
1553
1554    #[test]
1555    fn fold_unreadable_releases_run_dir_and_worktrees() {
1556        let dir = tempfile::tempdir().unwrap();
1557        let runs = dir.path().join("runs");
1558        let wt = dir.path().join("wt");
1559        let id = "20260905-000000-abcd";
1560        std::fs::create_dir_all(runs.join(id)).unwrap();
1561        std::fs::write(runs.join(id).join("run.json"), "garbage").unwrap();
1562        std::fs::create_dir_all(wt.join("abcd")).unwrap();
1563
1564        let removed = block_on(fold_unreadable(&runs, &wt, id)).expect("fold");
1565        assert_eq!(removed.len(), 2);
1566        assert!(!runs.join(id).exists(), "run dir gone");
1567        assert!(!wt.join("abcd").exists(), "worktrees gone");
1568
1569        // A prefix resolves like `run::resolve_id` does.
1570        std::fs::create_dir_all(runs.join(id)).unwrap();
1571        std::fs::write(runs.join(id).join("run.json"), "garbage").unwrap();
1572        std::fs::create_dir_all(wt.join("abcd")).unwrap();
1573        let removed = block_on(fold_unreadable(&runs, &wt, "20260905")).expect("by prefix");
1574        assert_eq!(removed.len(), 2);
1575        // Once gone, `id` cannot be resolved at all - same as `run::resolve_id`
1576        // on an id nothing on disk matches - so a repeat pass errors rather
1577        // than silently reporting nothing removed.
1578        assert!(
1579            block_on(fold_unreadable(&runs, &wt, id)).is_err(),
1580            "a run already gone cannot be resolved again"
1581        );
1582    }
1583
1584    #[test]
1585    fn prune_cache_sheds_the_oldest_generation_until_it_fits() {
1586        let home = tempfile::tempdir().unwrap();
1587        let dir = tempfile::tempdir().unwrap();
1588        // Same size, different age: only the age decides, and the newest
1589        // generation - the one the next build reuses - is what survives.
1590        fs::write(dir.path().join("old"), b"xx").unwrap();
1591        fs::write(dir.path().join("new"), b"yy").unwrap();
1592        touch(&dir.path().join("old"), 1_000_000);
1593        touch(&dir.path().join("new"), 2_000_000);
1594
1595        let out = prune_cache(home.path(), dir.path(), 2)
1596            .expect("prune")
1597            .expect("the cache is free");
1598        assert_eq!(out.files, 1, "one deletion is enough to reach the cap");
1599        assert_eq!(out.remaining, 2);
1600        assert!(!dir.path().join("old").exists(), "the older file went");
1601        assert!(dir.path().join("new").exists(), "the newer one stayed");
1602
1603        // A whole generation shares one timestamp tick, so the tie has to be
1604        // decided too: largest first, which reaches the cap in the fewest
1605        // deletions. Left to `read_dir` and an unstable sort this deleted
1606        // both files on Linux and one on Windows.
1607        let tied = tempfile::tempdir().unwrap();
1608        fs::write(tied.path().join("big"), b"xxxx").unwrap();
1609        fs::write(tied.path().join("small"), b"yy").unwrap();
1610        touch(&tied.path().join("big"), 1_000_000);
1611        touch(&tied.path().join("small"), 1_000_000);
1612        let out = prune_cache(home.path(), tied.path(), 2)
1613            .expect("prune")
1614            .expect("the cache is free");
1615        assert_eq!(out.files, 1, "the big one alone gets under the cap");
1616        assert_eq!(out.remaining, 2);
1617        assert!(tied.path().join("small").exists());
1618    }
1619
1620    #[test]
1621    fn prune_cache_if_over_limit_resolves_the_opt_outs_before_ever_measuring() {
1622        let home = tempfile::tempdir().unwrap();
1623        let dir = tempfile::tempdir().unwrap();
1624        fs::write(dir.path().join("big"), vec![0u8; 10]).unwrap();
1625
1626        let mut cfg = crate::config::Config::default();
1627        cfg.verify.gate = vec![format!(
1628            "CARGO_TARGET_DIR={} cargo make check",
1629            dir.path().display()
1630        )];
1631
1632        // A cap of `0` is the operator's opt-out: never measured, never
1633        // pruned, regardless of what is actually on disk.
1634        cfg.disk.cache_limit_bytes = 0;
1635        assert_eq!(
1636            prune_cache_if_over_limit(&cfg, home.path()).unwrap(),
1637            None,
1638            "a zero cap must not even look at the directory"
1639        );
1640        assert!(dir.path().join("big").exists());
1641
1642        // No `CARGO_TARGET_DIR` in either verify command: nothing to
1643        // aggregate, so there is nothing to prune either.
1644        let mut no_cache = crate::config::Config::default();
1645        no_cache.disk.cache_limit_bytes = 1;
1646        assert_eq!(
1647            prune_cache_if_over_limit(&no_cache, home.path()).unwrap(),
1648            None
1649        );
1650
1651        // Over the cap and configured: pruned exactly like `prune_cache`
1652        // itself would.
1653        cfg.disk.cache_limit_bytes = 1;
1654        let pruned = prune_cache_if_over_limit(&cfg, home.path())
1655            .unwrap()
1656            .expect("a real cache dir over its cap prunes");
1657        assert_eq!(pruned.files, 1);
1658        assert!(!dir.path().join("big").exists());
1659    }
1660
1661    /// Pin a file's mtime, so a test asserts the policy and not the runner's
1662    /// timestamp granularity.
1663    fn touch(path: &Path, secs: u64) {
1664        let f = fs::File::options().write(true).open(path).unwrap();
1665        f.set_times(fs::FileTimes::new().set_modified(
1666            std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(secs),
1667        ))
1668        .unwrap();
1669    }
1670
1671    /// The disk-full casualty: a run whose first save left `run.json.tmp` and
1672    /// nothing else. It has to be clearable, or the record is permanent.
1673    #[test]
1674    fn fold_unreadable_clears_a_run_whose_state_never_landed() {
1675        let dir = tempfile::tempdir().unwrap();
1676        let runs = dir.path().join("runs");
1677        let wt = dir.path().join("wt");
1678        let id = "20260904-014540-88c0";
1679        std::fs::create_dir_all(runs.join(id)).unwrap();
1680        std::fs::write(runs.join(id).join("run.json.tmp"), b"").unwrap();
1681
1682        let removed = block_on(fold_unreadable(&runs, &wt, id)).expect("fold by id");
1683        assert_eq!(removed, vec![format!("runs/{id}")]);
1684        assert!(!runs.join(id).exists(), "record gone");
1685
1686        // And by prefix, the way the deck and the phone address a run.
1687        std::fs::create_dir_all(runs.join(id)).unwrap();
1688        std::fs::write(runs.join(id).join("run.json.tmp"), b"").unwrap();
1689        assert!(
1690            block_on(fold_unreadable(&runs, &wt, "88c0")).is_ok(),
1691            "by prefix"
1692        );
1693
1694        // A directory under `runs` that is not a run is never a fold target.
1695        std::fs::create_dir_all(runs.join("scratch")).unwrap();
1696        assert!(
1697            block_on(fold_unreadable(&runs, &wt, "scratch")).is_err(),
1698            "a stray directory is not a run"
1699        );
1700    }
1701
1702    #[test]
1703    fn fold_due_folds_terminal_runs_of_any_schema_but_leaves_genuinely_unreadable_ones() {
1704        let dir = tempfile::tempdir().unwrap();
1705        let runs = dir.path().join("runs");
1706        let wt = dir.path().join("wt");
1707        let home = dir.path().to_path_buf();
1708        let disk = Disk::default();
1709        let now = ts("2026-09-05T00:00:00Z");
1710
1711        // 1. Runnable (judging): never folded, however old.
1712        let judging = "20260801-000000-0001";
1713        write_meta(&runs, judging, "judging", "2026-08-01T00:00:00Z");
1714
1715        // 2. Finished but fresh: grace not elapsed. Within the default 6h
1716        //    grace of `now`, so `fold_due` must stop at the freshness check
1717        //    and never even reach `read_state` - `write_meta`'s minimal JSON
1718        //    would fail that full parse anyway, and this case exists to
1719        //    prove freshness is why the run survives, not an accident of the
1720        //    fixture being unparseable as a whole `RunState`.
1721        let ready_fresh = "20260904-220000-0002";
1722        write_meta(&runs, ready_fresh, "ready", "2026-09-04T22:00:00Z");
1723
1724        // 3. Genuinely unreadable: broken JSON, not merely an unfamiliar
1725        //    schema number. Left alone and counted - this is the one case
1726        //    automatic housekeeping must never touch (see `fold_due`'s docs);
1727        //    discarding it is an explicit operator action, not something a
1728        //    background pass does.
1729        let garbage = "20260901-000000-0004";
1730        std::fs::create_dir_all(runs.join(garbage)).unwrap();
1731        std::fs::write(runs.join(garbage).join("run.json"), "not json").unwrap();
1732        std::fs::create_dir_all(wt.join("0004")).unwrap();
1733
1734        // 4. Finished, well past grace, current schema: the ordinary case
1735        //    `fold_due` has always acted on.
1736        let due_ready = due_run(&runs, &wt, "20260801-000000-ffff", SCHEMA);
1737
1738        // 5. Finished, well past grace, but written by a schema number this
1739        //    build no longer matches - the defect this task exists to fix.
1740        //    It still parses cleanly, so only the version number differs, and
1741        //    that alone must not block folding.
1742        let due_old_schema = due_run(&runs, &wt, "20260801-000000-eeee", SCHEMA - 1);
1743
1744        let (folded, unreadable) =
1745            block_on(fold_due(&runs, &home, &wt, &disk, now)).expect("fold_due");
1746        assert_eq!(
1747            folded, 2,
1748            "both due, parseable runs fold regardless of their schema number"
1749        );
1750        assert_eq!(
1751            unreadable, 1,
1752            "only the run with broken JSON counts as unreadable"
1753        );
1754        assert!(runs.join(judging).exists(), "runnable never folded");
1755        assert!(runs.join(ready_fresh).exists(), "fresh never folded");
1756        assert!(runs.join(garbage).exists(), "unreadable record kept");
1757        assert!(wt.join("0004").exists(), "unreadable worktree kept");
1758        assert!(
1759            runs.join(&due_ready).exists(),
1760            "folding drops worktrees, not the record"
1761        );
1762        assert!(
1763            runs.join(&due_old_schema).exists(),
1764            "an old-schema record survives its fold exactly like a current one"
1765        );
1766        // `graph::fold_run` saves the state it just folded back to disk. That
1767        // write must land under this test's own `runs` - the argument it
1768        // passed to `fold_due`, not the process-global `run::home` - or a
1769        // fold that landed somewhere else entirely would still be counted
1770        // above as one of the two `folded` runs.
1771        for id in [&due_ready, &due_old_schema] {
1772            let saved = read_meta(&runs, id).expect("folded run still parses");
1773            assert_ne!(
1774                saved.updated_at,
1775                ts("2026-08-01T00:00:00Z"),
1776                "fold_run must have saved the updated state back through the \
1777                 `runs` directory this test passed to fold_due"
1778            );
1779        }
1780    }
1781
1782    /// `[disk] auto_fold = false` must leave the janitor's fold-and-reclaim
1783    /// passes completely inert - a due run's worktree and record both
1784    /// survive exactly as if `housekeep` had never run at all. Cache pruning
1785    /// is a separate opt-out (`cache_limit_bytes`) and stays disabled here
1786    /// too, so this test is only ever about `auto_fold`.
1787    #[tokio::test]
1788    async fn housekeep_leaves_everything_alone_when_auto_fold_is_disabled() {
1789        let dir = tempfile::tempdir().unwrap();
1790        let runs = dir.path().join("runs");
1791        let wt = dir.path().join("wt");
1792        let home = dir.path().to_path_buf();
1793        crate::run::set_home(dir.path().to_path_buf());
1794
1795        let due_id = due_run(&runs, &wt, "20260801-000000-abcd", SCHEMA);
1796        std::fs::create_dir_all(wt.join("orphan").join("cand-A")).unwrap();
1797
1798        let mut cfg = crate::config::Config::default();
1799        cfg.disk.auto_fold = false;
1800        cfg.disk.cache_limit_bytes = 0;
1801
1802        let out = housekeep(&cfg, &home, &wt, &dir.path().join("repo"), Timestamp::now()).await;
1803
1804        assert_eq!(out.folded, 0);
1805        assert_eq!(out.unreadable, 0);
1806        assert_eq!(out.orphaned_worktrees, 0);
1807        assert!(
1808            runs.join(&due_id).exists(),
1809            "a due run's record survives untouched"
1810        );
1811        assert!(
1812            wt.join("orphan").exists(),
1813            "an orphaned worktree survives untouched: the reclaim pass never ran"
1814        );
1815    }
1816
1817    #[test]
1818    fn fold_orphaned_worktrees_removes_only_worktrees_no_run_claims_and_none_in_flight() {
1819        let dir = tempfile::tempdir().unwrap();
1820        let runs = dir.path().join("runs");
1821        let wt = dir.path().join("wt");
1822        let home = dir.path().to_path_buf();
1823
1824        // A run record exists for this one: its worktree is claimed, not
1825        // orphaned, however old the record.
1826        write_meta(
1827            &runs,
1828            "20260801-000000-aaaa",
1829            "ready",
1830            "2026-08-01T00:00:00Z",
1831        );
1832        std::fs::create_dir_all(wt.join("aaaa").join("cand-A")).unwrap();
1833
1834        // No run record at all, and nobody is working on it: this is the
1835        // leftover `fold_due` can never see, because it only ever walks
1836        // `runs/`.
1837        std::fs::create_dir_all(wt.join("bbbb").join("cand-A")).unwrap();
1838
1839        // No run record either, but a live daemon status names a run with
1840        // this short id - the save-timing gap between the daemon claiming a
1841        // task and `RunState::new` writing its first `run.json`. Must survive
1842        // untouched.
1843        std::fs::create_dir_all(wt.join("cccc")).unwrap();
1844
1845        // Not shaped like a run's short id at all - a scratch directory an
1846        // operator or another tool left in the same bay - so it is never a
1847        // reclaim target regardless of what runs claim it or not.
1848        std::fs::create_dir_all(wt.join("scratch")).unwrap();
1849
1850        // `now` pushed comfortably past `MIN_ORPHAN_AGE_SECS`, so a zero
1851        // grace - the same "always due" escape hatch `due` itself documents
1852        // - still reclaims once a worktree is genuinely old, without faking
1853        // an mtime: real directory creation just above is already in the
1854        // past relative to this `now`, by design rather than by timing.
1855        let now = Timestamp::now() + SignedDuration::new((MIN_ORPHAN_AGE_SECS + 1) as i64, 0);
1856        let mut status = crate::daemon::Status::new();
1857        status.current = vec![crate::daemon::Current {
1858            worker: None,
1859            task: "20260905-000000-t111".to_owned(),
1860            run: "20260905-000000-cccc".to_owned(),
1861        }];
1862        status.updated_at = now;
1863        crate::daemon::write_status_to(&home.join("daemon.json"), &status).unwrap();
1864
1865        let folded = block_on(fold_orphaned_worktrees(&runs, &wt, &home, 0, now));
1866        assert_eq!(
1867            folded, 1,
1868            "only the truly orphaned, idle, bay-shaped worktree is removed"
1869        );
1870        assert!(wt.join("aaaa").exists(), "claimed by a run record");
1871        assert!(!wt.join("bbbb").exists(), "orphaned and idle: reclaimed");
1872        assert!(wt.join("cccc").exists(), "a run in flight is never touched");
1873        assert!(
1874            wt.join("scratch").exists(),
1875            "not shaped like a worktree bay, so never a reclaim target"
1876        );
1877    }
1878
1879    /// The gap this closes: `Runner::review` (`magi review`) creates the
1880    /// worktree with `git worktree add` before `RunState::save` ever writes a
1881    /// `run.json`, and that path never runs through the daemon's own `poll`
1882    /// loop at all, so `daemon::Status` never names it either. Without a
1883    /// grace window, a janitor pass landing in that gap would read the
1884    /// worktree as an orphan nothing is waiting on and delete a review still
1885    /// being set up.
1886    #[test]
1887    fn fold_orphaned_worktrees_leaves_a_freshly_created_bay_alone() {
1888        let dir = tempfile::tempdir().unwrap();
1889        let runs = dir.path().join("runs");
1890        let wt = dir.path().join("wt");
1891        let home = dir.path().to_path_buf();
1892
1893        std::fs::create_dir_all(wt.join("dddd").join("under-review")).unwrap();
1894
1895        let now = Timestamp::now();
1896        let folded = block_on(fold_orphaned_worktrees(&runs, &wt, &home, 6 * 60 * 60, now));
1897        assert_eq!(
1898            folded, 0,
1899            "too fresh to tell apart from a run still being set up"
1900        );
1901        assert!(wt.join("dddd").exists());
1902    }
1903
1904    /// A fresh open question on `run`, stored and handed back for assertions.
1905    fn open_question(store: &Questions, run: &str) -> crate::ask::Question {
1906        let mut q = crate::ask::Question::new(
1907            run.to_owned(),
1908            "implement".to_owned(),
1909            "impl-A".to_owned(),
1910            "Which storage backend should the cache use?".to_owned(),
1911            String::new(),
1912            vec!["SQLite".to_owned(), "Redis".to_owned()],
1913        );
1914        store.put(&mut q).unwrap();
1915        q
1916    }
1917
1918    /// The exact ghost the phone showed: a run that already finished, with a
1919    /// question its dead seat asked still sitting `open` because it reached
1920    /// that status before `graph::Runner::settle_questions` existed (or
1921    /// missed it in the crash window `daemon::reclaim_orphaned_running`
1922    /// covers). This sweep is the second door to the same fact.
1923    #[test]
1924    fn a_finished_runs_open_question_is_swept_up() {
1925        let dir = tempfile::tempdir().unwrap();
1926        let runs = dir.path().join("runs");
1927        let store = Questions::at(dir.path().join("questions"));
1928
1929        let failed = "20260908-205802-c9eb";
1930        write_meta(&runs, failed, "failed", "2026-09-08T20:58:02Z");
1931        let failed_q = open_question(&store, failed);
1932
1933        let merged = "20260908-205501-ca67";
1934        write_meta(&runs, merged, "merged", "2026-09-08T20:55:01Z");
1935        let merged_q = open_question(&store, merged);
1936
1937        let n = abandon_settled_questions(&store, &runs);
1938        assert_eq!(n, 2, "both dead runs' questions are swept in one pass");
1939
1940        for (id, run) in [(&failed_q.id, failed), (&merged_q.id, merged)] {
1941            let back = store.get(id).unwrap();
1942            assert!(!back.status.open(), "{run} is done; nobody reads an answer");
1943            assert!(back.detail.contains(run), "{}", back.detail);
1944        }
1945    }
1946
1947    #[test]
1948    fn a_still_alive_runs_open_question_survives_the_sweep() {
1949        let dir = tempfile::tempdir().unwrap();
1950        let runs = dir.path().join("runs");
1951        let store = Questions::at(dir.path().join("questions"));
1952
1953        // `Blocked` and `Stalled` are `RunStatus::resumable`: the run can
1954        // still be picked back up, so its question may yet get a real
1955        // answer. A run still mid-competition is even more obviously alive.
1956        for (id, status) in [
1957            ("20260908-000000-b10c", "blocked"),
1958            ("20260908-000000-5ta1", "stalled"),
1959            ("20260908-000000-jud6", "judging"),
1960        ] {
1961            write_meta(&runs, id, status, "2026-09-08T00:00:00Z");
1962            let q = open_question(&store, id);
1963
1964            let n = abandon_settled_questions(&store, &runs);
1965            assert_eq!(n, 0, "{status} run is not done; nothing to sweep");
1966            assert!(
1967                store.get(&q.id).unwrap().status.open(),
1968                "{status} run's question must still be waiting"
1969            );
1970        }
1971    }
1972
1973    #[test]
1974    fn the_sweep_leaves_an_answered_question_and_an_unreadable_run_alone() {
1975        let dir = tempfile::tempdir().unwrap();
1976        let runs = dir.path().join("runs");
1977        let store = Questions::at(dir.path().join("questions"));
1978
1979        // Already decided: a sweep must never revisit it, whatever the run
1980        // that asked went on to become.
1981        let done = "20260908-000000-answ";
1982        write_meta(&runs, done, "failed", "2026-09-08T00:00:00Z");
1983        let mut answered = open_question(&store, done);
1984        answered
1985            .answer(crate::ask::Answer::Choice("SQLite".to_owned()))
1986            .unwrap();
1987        store.put(&mut answered).unwrap();
1988
1989        // No `run.json` at all for this one - deleted, or never landed.
1990        let gone = "20260908-000000-gone";
1991        let orphan = open_question(&store, gone);
1992
1993        assert_eq!(abandon_settled_questions(&store, &runs), 0);
1994        assert_eq!(
1995            store.get(&answered.id).unwrap().status,
1996            crate::ask::QuestionStatus::Answered,
1997            "a real answer is never overwritten by a sweep"
1998        );
1999        assert!(
2000            store.get(&orphan.id).unwrap().status.open(),
2001            "a run this sweep cannot read is left exactly as it was, not guessed at"
2002        );
2003    }
2004
2005    /// A grace of `0` is a legitimate, documented value for the operator's
2006    /// own `Disk::fold_grace_secs` - `due`'s "always due" case - but the
2007    /// freshness check this guards is not that policy, and must not collapse
2008    /// to it: a `0` handed straight through would reclaim a worktree the
2009    /// instant it exists, exactly the race `fold_orphaned_worktrees_leaves_a_
2010    /// freshly_created_bay_alone` exists to rule out, just with the operator
2011    /// having turned the other grace off instead of leaving it at its
2012    /// default.
2013    #[test]
2014    fn fold_orphaned_worktrees_floors_a_zero_grace_at_the_race_safe_minimum() {
2015        let dir = tempfile::tempdir().unwrap();
2016        let runs = dir.path().join("runs");
2017        let wt = dir.path().join("wt");
2018        let home = dir.path().to_path_buf();
2019
2020        std::fs::create_dir_all(wt.join("eeee").join("under-review")).unwrap();
2021
2022        // Too fresh, even with the grace argument at zero.
2023        let now = Timestamp::now();
2024        let folded = block_on(fold_orphaned_worktrees(&runs, &wt, &home, 0, now));
2025        assert_eq!(
2026            folded, 0,
2027            "a zero grace must not defeat the race-safety floor"
2028        );
2029        assert!(wt.join("eeee").exists());
2030
2031        // Once genuinely past the floor, a zero grace reclaims it - the
2032        // floor is a minimum, not a replacement policy that never fires.
2033        let later = now + SignedDuration::new((MIN_ORPHAN_AGE_SECS + 1) as i64, 0);
2034        let folded = block_on(fold_orphaned_worktrees(&runs, &wt, &home, 0, later));
2035        assert_eq!(folded, 1, "old enough now, regardless of the zero grace");
2036        assert!(!wt.join("eeee").exists());
2037    }
2038
2039    #[test]
2040    fn clear_abandoned_active_only_acts_once_dead_and_overrun() {
2041        let dir = tempfile::tempdir().unwrap();
2042        // Harmless if another test in this binary already pinned the global
2043        // home first (see `run::set_home`'s own doc): this test only checks
2044        // the in-memory mutation `clear_abandoned_active` makes, never a
2045        // write that landed under this exact directory.
2046        crate::run::set_home(dir.path().to_path_buf());
2047        let home = dir.path().to_path_buf();
2048        let now = ts("2026-09-14T12:00:00Z");
2049        let overrun_seat = || crate::run::ActiveSeat {
2050            node: "implement".to_owned(),
2051            started_at: now - SignedDuration::new(21_000, 0),
2052            timeout_secs: 3_600,
2053            attempt: 0,
2054            task: None,
2055            command: None,
2056            index: None,
2057            total: None,
2058        };
2059
2060        let mut state = RunState::new(
2061            PathBuf::from("/repo"),
2062            "main".to_owned(),
2063            "abc1234".to_owned(),
2064            "fixture".to_owned(),
2065            crate::config::Config::default(),
2066        );
2067        state.status = RunStatus::Implementing;
2068        state.active.insert("impl-A".to_owned(), overrun_seat());
2069
2070        // A seat still within its own budget: not provably dead yet, so this
2071        // must change nothing.
2072        let mut fresh = state.clone();
2073        fresh.active.insert(
2074            "impl-B".to_owned(),
2075            crate::run::ActiveSeat {
2076                node: "implement".to_owned(),
2077                started_at: now,
2078                timeout_secs: 3_600,
2079                attempt: 0,
2080                task: None,
2081                command: None,
2082                index: None,
2083                total: None,
2084            },
2085        );
2086        assert!(!clear_abandoned_active(&mut fresh, &home, now).unwrap());
2087        assert!(!fresh.active.is_empty());
2088        assert_eq!(fresh.status, RunStatus::Implementing);
2089
2090        let store = Questions::at(home.join("questions"));
2091        let q = open_question(&store, &state.id);
2092
2093        assert!(clear_abandoned_active(&mut state, &home, now).unwrap());
2094        assert!(state.active.is_empty());
2095        assert_eq!(state.status, RunStatus::Failed);
2096        assert!(
2097            !store.get(&q.id).unwrap().status.open(),
2098            "the abandoned seat's own open question must not keep badging the \
2099             operator until some later daemon startup notices it"
2100        );
2101    }
2102
2103    /// Write a whole `run.json` that magi can read, over the given state.
2104    fn write_meta(runs: &Path, id: &str, status: &str, updated_at: &str) {
2105        let day = &updated_at[..10];
2106        std::fs::create_dir_all(runs.join(id)).unwrap();
2107        let body = format!(
2108            r#"{{"schema": {SCHEMA}, "id": "{id}", "repo": "/nonexistent/repo", "base_branch": "main", "base_commit": "0000000000000000000000000000000000000000", "instruction": "", "created_at": "{day}T00:00:00Z", "updated_at": "{updated_at}", "status": "{status}", "seed": 1}}"#
2109        );
2110        std::fs::write(runs.join(id).join("run.json"), body).unwrap();
2111    }
2112
2113    /// Write a fully-formed, `Ready`, well-past-grace `run.json` tagged with
2114    /// an arbitrary schema number - so a test can write one this build's own
2115    /// `RunState::new` could never produce on its own. Returns the id.
2116    ///
2117    /// `wt` becomes this run's `graph.worktree_root`: left at the config
2118    /// default, `RunState::worktree_root` falls through to
2119    /// `run::default_worktree_root` - the operator's real `~/wt/magi` - and
2120    /// `graph::fold_run`'s second sweep would then `read_dir` and remove
2121    /// worktrees there instead of anything this test owns.
2122    fn due_run(runs: &Path, wt: &Path, id: &str, schema: u32) -> String {
2123        due_run_with_status(runs, wt, id, schema, RunStatus::Ready)
2124    }
2125
2126    /// [`due_run`], with the terminal status a caller wants instead of the
2127    /// `Ready` every other caller here happens to want.
2128    fn due_run_with_status(
2129        runs: &Path,
2130        wt: &Path,
2131        id: &str,
2132        schema: u32,
2133        status: RunStatus,
2134    ) -> String {
2135        let mut config = crate::config::Config::default();
2136        config.graph.worktree_root = Some(wt.to_path_buf());
2137        let mut state = RunState::new(
2138            PathBuf::from("/nonexistent/repo"),
2139            "main".to_owned(),
2140            "0000000000000000000000000000000000000000".to_owned(),
2141            String::new(),
2142            config,
2143        );
2144        state.id = id.to_owned();
2145        state.status = status;
2146        state.updated_at = ts("2026-08-01T00:00:00Z");
2147        let mut value = serde_json::to_value(&state).unwrap();
2148        value["schema"] = serde_json::json!(schema);
2149        std::fs::create_dir_all(runs.join(id)).unwrap();
2150        std::fs::write(
2151            runs.join(id).join("run.json"),
2152            serde_json::to_string_pretty(&value).unwrap(),
2153        )
2154        .unwrap();
2155        id.to_owned()
2156    }
2157
2158    #[test]
2159    fn fold_due_folds_a_superseded_run_same_as_any_other_terminal_one() {
2160        let dir = tempfile::tempdir().unwrap();
2161        let runs = dir.path().join("runs");
2162        let wt = dir.path().join("wt");
2163        let home = dir.path().to_path_buf();
2164        let disk = Disk::default();
2165        let now = ts("2026-09-05T00:00:00Z");
2166
2167        let superseded = due_run_with_status(
2168            &runs,
2169            &wt,
2170            "20260801-000000-cccc",
2171            SCHEMA,
2172            RunStatus::Superseded,
2173        );
2174
2175        let (folded, unreadable) =
2176            block_on(fold_due(&runs, &home, &wt, &disk, now)).expect("fold_due");
2177        assert_eq!(
2178            folded, 1,
2179            "a superseded run has nothing left for a human to check, so it folds \
2180             exactly like a merged one"
2181        );
2182        assert_eq!(unreadable, 0);
2183        assert!(
2184            read_meta(&runs, &superseded).is_ok(),
2185            "folding drops the worktree, not the record"
2186        );
2187    }
2188
2189    /// A throwaway repo with one commit on `main`, for a test that needs a
2190    /// real winner worktree `fold_run` can actually remove.
2191    fn init_repo(dir: &Path) {
2192        use crate::proc::Quiet as _;
2193        let run = |args: &[&str]| {
2194            let out = std::process::Command::new("git")
2195                .args(args)
2196                .current_dir(dir)
2197                .quiet()
2198                .output()
2199                .expect("spawn git");
2200            assert!(
2201                out.status.success(),
2202                "git {args:?} failed: {}",
2203                String::from_utf8_lossy(&out.stderr)
2204            );
2205        };
2206        run(&["init", "-b", "main"]);
2207        run(&["config", "user.name", "magi test"]);
2208        run(&["config", "user.email", "magi@example.com"]);
2209        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
2210        run(&["add", "-A"]);
2211        run(&["commit", "-m", "init"]);
2212    }
2213
2214    /// A due, terminal run with a decided winner that still has a real
2215    /// worktree and branch on disk - so a test can tell whether folding it
2216    /// actually removed the winner or only left it registered as folded.
2217    async fn due_run_with_winner_worktree(
2218        runs: &Path,
2219        wt: &Path,
2220        repo: &Path,
2221        id: &str,
2222        status: RunStatus,
2223    ) -> (String, PathBuf) {
2224        use crate::run::{Candidate, Tally};
2225        use std::collections::BTreeMap;
2226
2227        let mut config = crate::config::Config::default();
2228        config.graph.worktree_root = Some(wt.to_path_buf());
2229        let mut state = RunState::new(
2230            repo.to_path_buf(),
2231            "main".to_owned(),
2232            "0000000000000000000000000000000000000000".to_owned(),
2233            String::new(),
2234            config,
2235        );
2236        state.id = id.to_owned();
2237        state.status = status;
2238        state.updated_at = ts("2026-08-01T00:00:00Z");
2239
2240        let winner_wt = state.worktree_root().join("cand-A");
2241        crate::git::worktree_add_branch(repo, &winner_wt, &format!("magi/{id}/A"), "main")
2242            .await
2243            .expect("winner worktree");
2244
2245        state.candidates.push(Candidate {
2246            index: 0,
2247            label: 'A',
2248            agent: "agent".to_owned(),
2249            branch: format!("magi/{id}/A"),
2250            worktree: winner_wt.clone(),
2251            summary: String::new(),
2252            stat: String::new(),
2253            files: 0,
2254            commits: 0,
2255            empty: false,
2256            failed: None,
2257            verified_noop: None,
2258            duration_ms: 0,
2259            folded: false,
2260        });
2261        state.tally = Some(Tally {
2262            first_choice: BTreeMap::from([('A', 1)]),
2263            borda: BTreeMap::new(),
2264            winner: 'A',
2265            rankings: 1,
2266            unanimous_initial: true,
2267            deliberated: false,
2268            changed_votes: 0,
2269            unanimous_final: true,
2270            tie_break: None,
2271            judges: 1,
2272            present: 1,
2273            quorum: 1,
2274            met_quorum: true,
2275            uncontested: None,
2276        });
2277        std::fs::create_dir_all(runs.join(id)).unwrap();
2278        std::fs::write(
2279            runs.join(id).join("run.json"),
2280            serde_json::to_string_pretty(&state).unwrap(),
2281        )
2282        .unwrap();
2283        (id.to_owned(), winner_wt)
2284    }
2285
2286    #[tokio::test]
2287    async fn completed_task_retains_independent_historical_sources_without_a_merge_proof() {
2288        let dir = tempfile::tempdir().unwrap();
2289        let home = dir.path();
2290        let runs = home.join("runs");
2291        let wt = home.join("wt");
2292        let repo = home.join("repo");
2293        std::fs::create_dir_all(&repo).unwrap();
2294        init_repo(&repo);
2295        let queue = crate::queue::Queue::at(home.join("queue"));
2296        let mut task = crate::queue::Task::new(
2297            "completed".into(),
2298            "retain independent historical changes".into(),
2299            repo.clone(),
2300            crate::queue::Source::Human,
2301        );
2302        let mut sources = Vec::new();
2303        for (id, status) in [
2304            ("20260801-000000-old1", RunStatus::Superseded),
2305            ("20260801-000000-old2", RunStatus::Failed),
2306        ] {
2307            let (id, worktree) = due_run_with_winner_worktree(&runs, &wt, &repo, id, status).await;
2308            std::fs::write(worktree.join("independent.txt"), &id).unwrap();
2309            crate::git::git(&worktree, &["add", "independent.txt"])
2310                .await
2311                .unwrap();
2312            crate::git::git(
2313                &worktree,
2314                &["commit", "-m", "Independent historical change"],
2315            )
2316            .await
2317            .unwrap();
2318            let head = crate::git::rev_parse(&worktree, "HEAD").await.unwrap();
2319            let mut state = RunState::load_under(&id, home).unwrap();
2320            state.driver_exited = true;
2321            if status == RunStatus::Failed {
2322                state.tally = None;
2323            }
2324            state.save_under(home).unwrap();
2325            task.link_run(&id);
2326            sources.push((id, worktree, head));
2327        }
2328        // A newer empty attempt and task completion are no proof that either
2329        // independent historical implementation was incorporated.
2330        let mut latest = RunState::new(
2331            repo.clone(),
2332            "main".into(),
2333            crate::git::rev_parse(&repo, "main").await.unwrap(),
2334            task.instruction.clone(),
2335            crate::config::Config::default(),
2336        );
2337        latest.status = RunStatus::Ready;
2338        latest.driver_exited = true;
2339        latest.save_under(home).unwrap();
2340        task.start(latest.id.clone());
2341        task.succeed();
2342        queue.put(&mut task).unwrap();
2343
2344        let (folded, unreadable) = fold_due(
2345            &runs,
2346            home,
2347            &wt,
2348            &Disk::default(),
2349            ts("2026-09-05T00:00:00Z"),
2350        )
2351        .await
2352        .unwrap();
2353        assert_eq!((folded, unreadable), (0, 0));
2354        for (id, worktree, head) in sources {
2355            assert!(worktree.exists());
2356            assert_eq!(
2357                crate::git::rev_parse(&worktree, "HEAD").await.unwrap(),
2358                head
2359            );
2360            let mut state = RunState::load_under(&id, home).unwrap();
2361            assert!(!state.candidates[0].folded);
2362            let error = crate::graph::fold_run(&mut state, true, home)
2363                .await
2364                .unwrap_err();
2365            assert!(
2366                error
2367                    .to_string()
2368                    .contains(&format!("source of task {}", task.id))
2369            );
2370            assert_eq!(
2371                crate::git::rev_parse(&repo, &state.candidates[0].branch)
2372                    .await
2373                    .unwrap(),
2374                head
2375            );
2376            assert!(worktree.exists());
2377            assert!(runs.join(&id).join("run.json").exists());
2378        }
2379    }
2380
2381    #[tokio::test]
2382    async fn recovery_retains_historical_sources_after_success_and_automatic_cleanup() {
2383        let dir = tempfile::tempdir().unwrap();
2384        let home = dir.path();
2385        let runs = home.join("runs");
2386        let wt = home.join("wt");
2387        let repo = home.join("repo");
2388        std::fs::create_dir_all(&repo).unwrap();
2389        init_repo(&repo);
2390        let mut task = crate::queue::Task::new(
2391            "recover".into(),
2392            "keep measured behavior".into(),
2393            repo.clone(),
2394            crate::queue::Source::Human,
2395        );
2396        let mut sources = Vec::new();
2397        for (id, status) in [
2398            ("20260801-000000-old1", RunStatus::Blocked),
2399            ("20260801-000000-old2", RunStatus::Stalled),
2400        ] {
2401            let (id, worktree) = due_run_with_winner_worktree(&runs, &wt, &repo, id, status).await;
2402            let mut state = RunState::load_under(&id, home).unwrap();
2403            state.driver_exited = true;
2404            state.save_under(home).unwrap();
2405            task.link_run(&id);
2406            sources.push((id, worktree));
2407        }
2408        crate::pr_recovery::retain_sources(&task, home).unwrap();
2409        let mut recovered = RunState::new(
2410            repo.clone(),
2411            "main".into(),
2412            "base".into(),
2413            task.instruction.clone(),
2414            crate::config::Config::default(),
2415        );
2416        recovered.config.graph.worktree_root = Some(wt.clone());
2417        recovered.status = RunStatus::Ready;
2418        recovered.driver_exited = true;
2419        recovered.save_under(home).unwrap();
2420        task.start(recovered.id.clone());
2421        task.succeed();
2422        crate::daemon::supersede_prior_runs(&task, home);
2423        for (id, _) in &sources {
2424            assert_eq!(
2425                RunState::load_under(id, home).unwrap().status,
2426                RunStatus::Superseded
2427            );
2428        }
2429        let disk = Disk::default();
2430        let now = Timestamp::now()
2431            .checked_add(SignedDuration::from_secs(
2432                i64::try_from(disk.fold_grace_secs + 1).unwrap(),
2433            ))
2434            .unwrap();
2435        let (_, unreadable) = fold_due(&runs, home, &wt, &disk, now).await.unwrap();
2436        assert_eq!(unreadable, 0);
2437        for (id, worktree) in sources {
2438            assert!(worktree.exists());
2439            assert!(
2440                crate::git::branch_exists(&repo, &format!("magi/{id}/A"))
2441                    .await
2442                    .unwrap()
2443            );
2444            let mut historical = RunState::load_under(&id, home).unwrap();
2445            assert!(!historical.candidates[0].folded);
2446            let error = crate::graph::fold_run(&mut historical, true, home)
2447                .await
2448                .unwrap_err();
2449            assert!(
2450                error
2451                    .to_string()
2452                    .contains("explicitly retained recovery source")
2453            );
2454            assert!(!historical.candidates[0].folded);
2455            assert!(
2456                worktree.exists(),
2457                "every automatic fold entry preserves retained sources"
2458            );
2459        }
2460    }
2461
2462    #[tokio::test]
2463    async fn fold_due_drops_a_superseded_runs_own_winner_worktree_but_keeps_a_readys() {
2464        let dir = tempfile::tempdir().unwrap();
2465        let runs = dir.path().join("runs");
2466        let wt = dir.path().join("wt");
2467        let repo = dir.path().join("repo");
2468        let home = dir.path().to_path_buf();
2469        let disk = Disk::default();
2470        let now = ts("2026-09-05T00:00:00Z");
2471        std::fs::create_dir_all(&repo).unwrap();
2472        init_repo(&repo);
2473
2474        let (superseded_id, superseded_wt) = due_run_with_winner_worktree(
2475            &runs,
2476            &wt,
2477            &repo,
2478            "20260801-000000-supw",
2479            RunStatus::Superseded,
2480        )
2481        .await;
2482        // `Ready` is auto-folded exactly like `Superseded` (both terminal and
2483        // non-resumable), but never landed anywhere - its own winner is
2484        // deliberately kept, which is what makes it the right contrast here.
2485        // `Blocked`/`Stalled` are `resumable()` and so never even reach
2486        // `fold_run` in the first place; they would not exercise the
2487        // `drop_winner` choice this test is about.
2488        let (ready_id, ready_wt) = due_run_with_winner_worktree(
2489            &runs,
2490            &wt,
2491            &repo,
2492            "20260801-000000-rdyw",
2493            RunStatus::Ready,
2494        )
2495        .await;
2496
2497        let (folded, unreadable) = fold_due(&runs, &home, &wt, &disk, now)
2498            .await
2499            .expect("fold_due");
2500        assert_eq!(folded, 2);
2501        assert_eq!(unreadable, 0);
2502
2503        assert!(
2504            !superseded_wt.exists(),
2505            "a superseded run's own winner never lands anywhere else, so its worktree \
2506             must be dropped exactly like a merged run's"
2507        );
2508        assert!(
2509            ready_wt.exists(),
2510            "a still-ready run's winner may yet be merged by hand - folding must not \
2511             touch it"
2512        );
2513
2514        // Both records survive the fold; only the worktrees differ.
2515        assert!(read_meta(&runs, &superseded_id).is_ok());
2516        assert!(read_meta(&runs, &ready_id).is_ok());
2517    }
2518}