Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => {
1256                worktree_setup::prepare_with(
1257                    &state.config,
1258                    repo,
1259                    &worktree,
1260                    crate::build_target::Coordination::Held(_coordination.as_ref()),
1261                )
1262                .await
1263            }
1264            Err(e) => Err(e),
1265        };
1266        if let Err(e) = setup {
1267            git::worktree_remove_clean(repo, &worktree).await.ok();
1268            return Err(e);
1269        }
1270        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1271            .await
1272            .unwrap_or(0);
1273        if commits == 0 {
1274            git::worktree_remove_clean(repo, &worktree).await.ok();
1275            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1276        }
1277        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1278            .await
1279            .map(|f| f.len())
1280            .unwrap_or(0);
1281        if files == 0
1282            && let (Ok(head_tree), Ok(base_tree)) = (
1283                git::tree_of(&worktree, "HEAD").await,
1284                git::tree_of(&worktree, &base_commit).await,
1285            )
1286            && head_tree == base_tree
1287        {
1288            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1289            git::worktree_remove_clean(repo, &worktree).await.ok();
1290            bail!(
1291                "`{branch}` at {} has a tree identical to base {}; this usually means \
1292                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1293                 against `{}/{branch}`) rather than an empty change",
1294                short(&head),
1295                short(&base_commit),
1296                state.config.merge.remote
1297            );
1298        }
1299        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1300            .await
1301            .unwrap_or_default();
1302
1303        state.candidates.push(Candidate {
1304            index: 0,
1305            label: 'A',
1306            // Not an agent id on purpose: nothing in the roster wrote this, and
1307            // the stats tables must not credit anyone with a win for it.
1308            agent: EXISTING_BRANCH.to_owned(),
1309            branch: review_branch,
1310            worktree,
1311            summary: String::new(),
1312            stat,
1313            files,
1314            commits,
1315            empty: false,
1316            failed: None,
1317            verified_noop: None,
1318            duration_ms: 0,
1319            folded: false,
1320        });
1321        state.tally = Some(Tally {
1322            first_choice: BTreeMap::from([('A', 0)]),
1323            borda: BTreeMap::new(),
1324            winner: 'A',
1325            rankings: 0,
1326            unanimous_initial: false,
1327            deliberated: false,
1328            changed_votes: 0,
1329            unanimous_final: false,
1330            tie_break: None,
1331            // No panel sat, so no quorum applies. Zero judges is the correct
1332            // number for work that never competed, and must not be reported as
1333            // a collapsed panel.
1334            judges: 0,
1335            present: 0,
1336            quorum: 0,
1337            met_quorum: true,
1338            uncontested: Some("review-only run: nothing competed".to_owned()),
1339        });
1340        state.status = RunStatus::Reviewing;
1341        state.prep_complete = Some(true);
1342        state.event(
1343            "start",
1344            format!(
1345                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1346                state.id
1347            ),
1348        );
1349        Ok(())
1350    }
1351
1352    /// Authorize one bounded synchronization interval, under the caller's queue task
1353    /// claim and the same owner/execution locks used by run maintenance.
1354    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1355        Self::authorize_base_recovery_request(home, id, task, false, None).await
1356    }
1357
1358    /// Queue release uses its durable approval generation exactly once.
1359    pub(crate) async fn authorize_base_recovery_from_resumption(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363    ) -> Result<()> {
1364        Self::authorize_base_recovery_request(home, id, task, true, None).await
1365    }
1366
1367    pub(crate) async fn authorize_base_recovery_control(
1368        home: &Path,
1369        task: &str,
1370        control: &mut crate::base_recovery_control::Transaction,
1371    ) -> Result<()> {
1372        let id = control.request.run.clone();
1373        Self::authorize_base_recovery_request(home, &id, task, false, Some(control)).await
1374    }
1375
1376    async fn authorize_base_recovery_request(
1377        home: &Path,
1378        id: &str,
1379        task: &str,
1380        resumption: bool,
1381        mut control: Option<&mut crate::base_recovery_control::Transaction>,
1382    ) -> Result<()> {
1383        let _owner = crate::pr_head::owner_lock(home, id)?;
1384        let mut state = RunState::load_under(id, home)?;
1385        crate::pr_head::ensure_owner(&state)?;
1386        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1387        anyhow::ensure!(
1388            queued.execution_runs().last().is_some_and(|run| run == id)
1389                && queued.status != crate::queue::TaskStatus::Done
1390                && !queued.fresh_start
1391                && queued.review_branch.is_none(),
1392            "run task ownership changed"
1393        );
1394        anyhow::ensure!(
1395            state
1396                .origin
1397                .as_ref()
1398                .and_then(|o| o.task.as_deref())
1399                .is_none_or(|owner| owner == task),
1400            "run belongs to another task"
1401        );
1402        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1403        anyhow::ensure!(
1404            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1405            "run is live or its ownership is unknown"
1406        );
1407        // Judge the previous driver's lease before acquiring our maintenance
1408        // lease. The owner lock excludes new driver claims across both steps.
1409        let _execution = crate::driver::maintenance(home, id)?;
1410        anyhow::ensure!(
1411            state.status.resumable()
1412                && state.released_to.is_none()
1413                && state.retired_to.is_none()
1414                && state.pr_taken_over_by.is_none(),
1415            "run ownership is final"
1416        );
1417        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1418        anyhow::ensure!(
1419            !state.pr_publication_pending
1420                && !crate::handover::landing_checkpoint(&state)
1421                && state.merge_decision.is_none(),
1422            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1423        );
1424        let winner = state.winner().context("run has no retained winner")?;
1425        anyhow::ensure!(
1426            git::is_clean(&winner.worktree).await?,
1427            "retained worktree has uncommitted changes"
1428        );
1429        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1430        anyhow::ensure!(
1431            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1432            "retained worktree does not match the winner branch"
1433        );
1434        if let Some(control) = control.as_ref() {
1435            anyhow::ensure!(
1436                head.starts_with(&control.request.head),
1437                "retained HEAD changed since the owner request"
1438            );
1439            anyhow::ensure!(
1440                state
1441                    .base_sync_recovery
1442                    .stop
1443                    .as_ref()
1444                    .is_none_or(|stop| stop.head == head),
1445                "retained HEAD differs from the recorded stop head; inspect it first"
1446            );
1447        }
1448        state.ensure_base_episode();
1449        state.record_base_sync_stop(&head);
1450        anyhow::ensure!(
1451            state
1452                .base_sync_recovery
1453                .stop
1454                .as_ref()
1455                .is_some_and(|s| s.kind == crate::base_sync_recovery::StopKind::Exhausted),
1456            "base recovery refused: missing eligible movement-exhaustion evidence; inspect with `magi task inspect-base-stop {task} --run {id}`. Textual stops require an evidence-bound base-conflict grant"
1457        );
1458        let fetched =
1459            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1460        anyhow::ensure!(
1461            fetched.ok(),
1462            "cannot authorize base recovery without a fetched base"
1463        );
1464        let approved_tip = git::rev_parse(
1465            &state.repo,
1466            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1467        )
1468        .await?;
1469        if let Some(control) = control.as_deref_mut() {
1470            let queue = crate::queue::Queue::at(home.join("queue"));
1471            control.prepare(&queue, &state, &queued, &head, &approved_tip)?;
1472            if let Some(prior) = state
1473                .base_sync_recovery
1474                .authorizations
1475                .iter()
1476                .find(|a| a.request.as_deref() == Some(&control.key()))
1477            {
1478                anyhow::ensure!(
1479                    !prior.consumed && prior.invalidated.is_none() && state.base_recovery_pending(),
1480                    "recovery request was consumed or invalidated; it cannot queue again"
1481                );
1482                return control.enqueue(&queue);
1483            }
1484        }
1485        if resumption {
1486            anyhow::ensure!(
1487                crate::daemon::compatible_run_instruction(&state, &queued),
1488                "resumption instruction does not match the retained run"
1489            );
1490            let approval = queued
1491                .resumption
1492                .as_ref()
1493                .context("no operator resumption")?;
1494            anyhow::ensure!(
1495                approval.instruction == queued.instruction,
1496                "resumption approval does not match the current task instruction"
1497            );
1498            state.authorize_base_sync_resumption(task, approval)?;
1499        } else {
1500            state.authorize_base_sync(task)?;
1501        }
1502        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1503        state
1504            .base_sync_recovery
1505            .authorizations
1506            .last_mut()
1507            .unwrap()
1508            .scope = Some(crate::base_sync_recovery::Scope {
1509            tip: approved_tip,
1510            instruction: queued.instruction.clone(),
1511            episode,
1512        });
1513        if let Some(control) = control.as_deref_mut() {
1514            let authorization = state.base_sync_recovery.authorizations.last_mut().unwrap();
1515            authorization.request = Some(control.key());
1516            authorization.generation = control.generation();
1517            authorization.decision = control.decision();
1518            authorization.question = control
1519                .request
1520                .decision
1521                .as_ref()
1522                .map(|_| control.request.request.clone());
1523        }
1524        state.event(
1525            "base-recovery",
1526            "owner authorized a bounded base synchronization interval; fresh verification required",
1527        );
1528        state.save_under(home)?;
1529        if let Some(control) = control {
1530            control.enqueue(&crate::queue::Queue::at(home.join("queue")))?;
1531        }
1532        Ok(())
1533    }
1534
1535    /// Reopen an existing run.
1536    pub fn resume(id: &str) -> Result<Self> {
1537        let state = RunState::load(id)?;
1538        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1539        if state.startup.as_ref().is_some_and(|s| !s.complete)
1540            || (state.status == RunStatus::Prep
1541                && state.instruction.trim().is_empty()
1542                && state.candidates.is_empty()
1543                && state.prep_complete == Some(false))
1544        {
1545            bail!(
1546                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1547            );
1548        }
1549        crate::pr_head::ensure_owner(&state)?;
1550        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1551            bail!(
1552                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1553            );
1554        }
1555        if let Some(to) = &state.released_to {
1556            bail!(
1557                "run {} cannot be resumed: its worktree was released to run {}",
1558                state.short(),
1559                crate::run::short_of(to)
1560            );
1561        }
1562        let roles = state.config.resolve_roles()?;
1563        if let Some(id) = &state.config.graph.selected_implementer
1564            && state.candidates.iter().any(|c| c.agent != *id)
1565        {
1566            bail!(
1567                "replacement implementer pin does not match the saved candidates; task must stay held"
1568            );
1569        }
1570        let max_parallel = state.config.graph.max_parallel.max(1);
1571        Ok(Self {
1572            state,
1573            roles,
1574            sem: Arc::new(Semaphore::new(max_parallel)),
1575            pause: Pause::new(),
1576            interrupt: Pause::new(),
1577        })
1578    }
1579
1580    /// Walk the graph to a terminal state, skipping nodes already recorded.
1581    ///
1582    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1583    /// the phone - ends here, so this is the one place a run that ended
1584    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1585    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1586    pub fn execute(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
1587        // Allocate before polling: boxing inside an async wrapper would still
1588        // retain its construction frame while the nested graph is polled.
1589        Box::pin(self.execute_inner())
1590    }
1591
1592    async fn execute_inner(&mut self) -> Result<()> {
1593        anyhow::ensure!(
1594            self.state.startup.as_ref().is_none_or(|s| s.complete),
1595            "startup incomplete; inspect retained evidence before explicit recovery"
1596        );
1597        if let Some(selection) = &self.state.pr_recovery
1598            && (selection.source_proof.is_some() || selection.source_run.is_some())
1599        {
1600            Box::pin(selection.verify_source(&self.state.repo)).await?;
1601        }
1602        // Queue claims cover daemon work; this also serializes manual drivers
1603        // consuming a repair answer, including the new review and decision.
1604        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1605            || self.state.land_approval.is_some()
1606            || self.state.merge_decision.is_some()
1607            || !self.state.merge_repairs.is_empty()
1608            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1609        {
1610            let claim = FixClaim::acquire(&self.state.dir())?;
1611            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1612            let disk = RunState::load(&self.state.id)?;
1613            let checkpoint = |s: &RunState| {
1614                (
1615                    s.merge_repairs.len(),
1616                    s.merge_repairs
1617                        .last()
1618                        .map(|r| (r.question.clone(), r.phase)),
1619                    s.land_approval
1620                        .as_ref()
1621                        .map(|a| (a.question.clone(), a.head.clone())),
1622                )
1623            };
1624            if checkpoint(&disk) != checkpoint(&self.state) {
1625                bail!(
1626                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1627                );
1628            }
1629            Some(claim)
1630        } else {
1631            None
1632        };
1633        let mut execution = self.claim_driver().await?;
1634        let runner = &mut *execution.runner;
1635        let wait = runner.repository_wait();
1636        // Keep the graph's large future off the callers' stack, including
1637        // standalone task attempts running on a default-sized test thread.
1638        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1639        let result = match result {
1640            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1641                record_repository_block(&mut runner.state, &e);
1642                runner.park_here()?;
1643                Ok(())
1644            }
1645            other => other,
1646        };
1647        execution.driver.finish(&mut runner.state).await?;
1648        let ended = if result.is_err() {
1649            Some({
1650                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1651                if let Err(error) = &result {
1652                    notice.message = format!(
1653                        "Run {} stopped with an error: {}",
1654                        runner.state.id,
1655                        error.to_string().lines().next().unwrap_or("unknown error")
1656                    );
1657                }
1658                notice
1659            })
1660        } else {
1661            crate::notices::run_ended(&runner.state)
1662        };
1663        if let Some(notice) = ended {
1664            crate::notices::raise(notice);
1665        }
1666        result
1667    }
1668
1669    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1670        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1671    }
1672
1673    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1674        let wait = self.repository_wait();
1675        coordinate_operation(&mut self.state, cwd, &wait).await
1676    }
1677
1678    /// Validate ownership and persist the driver claim in one critical section.
1679    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1680        let owner = match crate::pr_head::owner_lock_wait(
1681            &crate::run::home(),
1682            &self.state.id,
1683            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1684        )
1685        .await
1686        {
1687            Ok(owner) => owner,
1688            Err(error) => {
1689                let detail = format!("worktree maintenance or ownership update: {error:#}");
1690                // Admission did not own run.json: retain this resource outcome
1691                // in memory for queue settlement without overwriting a concurrent
1692                // retirement or publication transaction.
1693                self.state.resource_block = Some(crate::run::ResourceBlock {
1694                    cause: crate::run::ResourceBlockCause::Other,
1695                    node: "start".into(),
1696                    kind: "worktree maintenance".into(),
1697                    detail: detail.clone(),
1698                    at: Timestamp::now(),
1699                });
1700                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1701            }
1702        };
1703        let disk = RunState::load(&self.state.id)?;
1704        crate::pr_head::ensure_owner(&disk)?;
1705        let claimed =
1706            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1707        // Only proven-live ownership refuses here. Unknown records (legacy runs
1708        // with no claim marker or process identity) fall through to the
1709        // exclusive execution lease taken just below, which is what actually
1710        // keeps a second driver out.
1711        anyhow::ensure!(
1712            disk.liveness(claimed) != crate::run::Liveness::Live,
1713            "run already has a live driver; wait for its execution to end"
1714        );
1715        if let Some(to) = &disk.released_to {
1716            bail!(
1717                "run {} cannot continue: its worktree was released to run {}",
1718                self.state.short(),
1719                crate::run::short_of(to)
1720            );
1721        }
1722        anyhow::ensure!(
1723            serde_json::to_value(&disk.base_sync_recovery)?
1724                == serde_json::to_value(&self.state.base_sync_recovery)?,
1725            "base recovery changed since this run was loaded; reload the saved run"
1726        );
1727        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1728        let mut execution = DriverExecution {
1729            runner: self,
1730            driver,
1731        };
1732        // Declare after the execution guard: on any claim error this short
1733        // lock must drop before that guard reloads and releases ownership.
1734        let _owner = owner;
1735        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1736            .await?;
1737        let state = &mut execution.runner.state;
1738        // Moving again, so it is no longer parked. Set before the walk rather
1739        // than in `resume`, so every way of re-entering the graph clears it
1740        // and a card cannot claim a run is waiting to be resumed while the
1741        // agents are already working.
1742        state.parked = false;
1743        state.resource_wait = None;
1744        state.resource_block = None;
1745        // Any seat this state still lists as answering belongs to whatever
1746        // process last drove this run — this one included, if it crashed
1747        // mid-wave. Cleared and flushed immediately, before anything else
1748        // runs, so a resume can never show a seat as live when nothing is
1749        // asking it anything yet; the node that actually dispatches the next
1750        // wave repopulates it.
1751        state.clear_active();
1752        // Recorded in the same spot, and flushed together with the clear
1753        // above: this is the pid a reader checks (`RunState::liveness`) when
1754        // no daemon claim exists to answer "is a process still driving this
1755        // run" — a plain `magi run` / `magi review` typed into a terminal
1756        // claims nothing there. Always overwritten, never only-if-absent, so
1757        // a resumed run's stale pid from a previous, possibly-dead process
1758        // can never survive into this one's own report. Unlike
1759        // `clear_active`, this changes on every single `execute()` call, so
1760        // the save below is now unconditional rather than only-if-cleared.
1761        //
1762        // `driver_started_at` is recorded in the same breath, from this same
1763        // pid, so `liveness` can tell a live pid that is genuinely still us
1764        // apart from one the OS has since handed to an unrelated process —
1765        // see that field's own doc for why the pid alone is not enough.
1766        let pid = std::process::id();
1767        state.driver_pid = Some(pid);
1768        state.driver_started_at = crate::proc::process_started_at(pid);
1769        state.driver_exited = false;
1770        state.driver_claimed = Some(true);
1771        state.begin_pass();
1772        execution.driver.bind(state);
1773        state.save()?;
1774        execution.driver.record(state)?;
1775        Ok(execution)
1776    }
1777
1778    /// No conflict checkpoint proves dispatch was never reached. End that
1779    /// consumed generation so an owner can explicitly authorize a fresh one.
1780    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1781        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1782        if let Err(error) = &result
1783            && !error.is::<crate::repo_lock::ResourceBlocked>()
1784            && self
1785                .state
1786                .repair_grants
1787                .last()
1788                .is_some_and(|g| g.conflict.is_none())
1789        {
1790            self.state.repair_grants.last_mut().unwrap().phase =
1791                crate::repair_budget::Phase::Complete;
1792            self.state.status = RunStatus::Blocked;
1793            self.state.save()?;
1794            bail!(
1795                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1796                self.state.passes.last().map_or(0, |p| p.index)
1797            );
1798        }
1799        result
1800    }
1801
1802    /// A grant owns only these repair attempts. Confirmation reviews cannot
1803    /// create another fixer allowance, and every operation is restart-checkpointed.
1804    async fn execute_granted_repair(&mut self) -> Result<()> {
1805        use crate::repair_budget::{Kind, Phase};
1806        let index = self.state.repair_grants.len() - 1;
1807        let winner = self
1808            .state
1809            .winner()
1810            .cloned()
1811            .context("repair winner missing")?;
1812        loop {
1813            let grant = self.state.repair_grants[index].clone();
1814            match grant.phase {
1815                Phase::Pending => {
1816                    if grant.spent >= grant.receipt.request.rounds {
1817                        self.state.repair_grants[index].phase = Phase::Complete;
1818                        self.state.status = RunStatus::Blocked;
1819                        self.state.save()?;
1820                        return Ok(());
1821                    }
1822                    // Reserve before dispatch. A crash can consume an attempt,
1823                    // but can never duplicate it or reset saved fixer history.
1824                    self.state.repair_grants[index].spent += 1;
1825                    self.state.repair_grants[index].phase = Phase::Fixing;
1826                    self.state.save()?;
1827                    match grant.receipt.request.kind {
1828                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1829                        Kind::Gate => {
1830                            let failed = self.state.gate.clone();
1831                            let _ = self.gate_fix_round(&winner, &failed).await?;
1832                        }
1833                        Kind::Review => {
1834                            let mut record = self
1835                                .state
1836                                .reviews
1837                                .last()
1838                                .cloned()
1839                                .context("review failure missing")?;
1840                            let round = record.round;
1841                            let step = self.run_fix_round(&winner, &mut record).await?;
1842                            self.state.reviews[round - 1] = record;
1843                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1844                                self.state.repair_grants[index].phase = Phase::Complete;
1845                                self.state.status = RunStatus::Blocked;
1846                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1847                                self.state.save()?;
1848                                return Ok(());
1849                            }
1850                            if matches!(step, FixStep::Resource) {
1851                                self.stop_for_resource()?;
1852                                return Ok(());
1853                            }
1854                            self.state.save()?;
1855                        }
1856                    }
1857                    if self.resource_halt()? {
1858                        return Ok(());
1859                    }
1860                }
1861                Phase::Fixing => {
1862                    if grant.receipt.request.kind == Kind::BaseConflict {
1863                        self.repair_base_conflict(false).await?;
1864                    }
1865                    // Interrupted fixers may have left source changes or an
1866                    // unpushed commit. Preserve them before fresh assessment.
1867                    let _coordination = self.coordinate(&winner.worktree).await?;
1868                    let rescued = git::rescue_commit(
1869                        &winner.worktree,
1870                        "magi: preserve interrupted granted repair",
1871                    )
1872                    .await?;
1873                    self.state.note_withheld("repair-grant", &rescued.withheld);
1874                    // Fresh checks must supersede even a same-head failure.
1875                    // Archive rather than erase failure evidence or counters.
1876                    self.state
1877                        .verification_history
1878                        .push(crate::run::VerificationEvidence {
1879                            superseded_by: None,
1880                            archived_at: Some(Timestamp::now()),
1881                            fixes_spent: Some(self.state.gate_fixes.len()),
1882                            gate_head: self.state.gate_head.take(),
1883                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1884                            gate: std::mem::take(&mut self.state.gate),
1885                            e2e: self.state.head_e2e.take(),
1886                        });
1887                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1888                        self.state
1889                            .verification_history
1890                            .push(crate::run::VerificationEvidence {
1891                                superseded_by: None,
1892                                archived_at: Some(Timestamp::now()),
1893                                fixes_spent: Some(self.state.gate_fixes.len()),
1894                                gate_head: None,
1895                                gate_ran: false,
1896                                gate: Vec::new(),
1897                                e2e: Some(e2e),
1898                            });
1899                    }
1900                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1901                    self.state.repair_grants[index].head =
1902                        git::rev_parse(&winner.worktree, "HEAD").await?;
1903                    self.state.repair_grants[index].phase = Phase::Reviewing;
1904                    if grant.receipt.request.kind == Kind::BaseConflict {
1905                        self.state.base_sync_recovery.stop = None;
1906                        if let Some(sync) = &mut self.state.base_sync {
1907                            sync.tip = grant
1908                                .receipt
1909                                .request
1910                                .tip
1911                                .clone()
1912                                .context("repair base tip missing")?;
1913                            sync.conflict = None;
1914                            sync.behind = 0;
1915                        }
1916                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1917                    }
1918                    self.state.save()?;
1919                }
1920                Phase::Reviewing => {
1921                    self.review_loop_pass().await?;
1922                    if self.resource_halt()? {
1923                        return Ok(());
1924                    }
1925                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1926                    let clean = self.state.reviews.len() > grant.review_start
1927                        && self.state.reviews.last().is_some_and(|r| {
1928                            r.head == head
1929                                && r.clean
1930                                && r.expected > 0
1931                                && r.answered == r.expected
1932                                && matches!(
1933                                    r.e2e_status(),
1934                                    E2eStatus::Passed | E2eStatus::NotConfigured
1935                                )
1936                        });
1937                    if !clean {
1938                        // A confirmation review carries no additional fix budget.
1939                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1940                            == Kind::Review
1941                            && grant.spent < grant.receipt.request.rounds
1942                            && self.state.reviews.last().is_some_and(|r| {
1943                                r.fix.is_none()
1944                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1945                            }) {
1946                            Phase::Pending
1947                        } else {
1948                            Phase::Complete
1949                        };
1950                        self.state.status = RunStatus::Blocked;
1951                        self.state.event(
1952                            "repair-grant",
1953                            "fresh review did not clear the repaired head; no publication",
1954                        );
1955                        self.state.save()?;
1956                        if self.state.repair_grants[index].phase == Phase::Pending {
1957                            continue;
1958                        }
1959                        return Ok(());
1960                    }
1961                    self.state.repair_grants[index].phase = Phase::Gating;
1962                    self.state.save()?;
1963                }
1964                Phase::Gating => {
1965                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1966                    self.state.reconcile_verification_head(&head);
1967                    let wait = self.repository_wait();
1968                    if self
1969                        .state
1970                        .reviews
1971                        .last()
1972                        .is_none_or(|r| r.head != head || !r.clean)
1973                    {
1974                        self.state.status = RunStatus::Blocked;
1975                        self.state.repair_grants[index].phase = Phase::Complete;
1976                        self.state.event(
1977                            "repair-grant",
1978                            "head changed since fresh review; repair held",
1979                        );
1980                        self.state.save()?;
1981                        return Ok(());
1982                    }
1983                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1984                        return Ok(());
1985                    }
1986                    let outcomes = self.run_gate(&winner).await?;
1987                    if verify_inconclusive(&outcomes) {
1988                        return Ok(());
1989                    }
1990                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1991                    if after != head {
1992                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1993                        self.state.repair_grants[index].head = after;
1994                        self.state.repair_grants[index].phase = Phase::Reviewing;
1995                        self.state.status = RunStatus::Blocked;
1996                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1997                        self.state.save()?;
1998                        return Ok(());
1999                    }
2000                    let passed = outcomes.iter().all(CommandOutcome::ok);
2001                    // Follow-up grants owe the same guarded checkout delivery
2002                    // as the conflict repair that introduced the new head.
2003                    if passed
2004                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
2005                    {
2006                        return Ok(());
2007                    }
2008                    self.state.repair_grants[index].succeeded = passed;
2009                    self.state.gate = outcomes;
2010                    self.state.gate_ran = true;
2011                    self.state.status = if passed {
2012                        RunStatus::Gating
2013                    } else {
2014                        RunStatus::Blocked
2015                    };
2016                    self.state.repair_grants[index].phase = if !passed
2017                        && grant.receipt.request.kind == Kind::Gate
2018                        && grant.spent < grant.receipt.request.rounds
2019                    {
2020                        Phase::Pending
2021                    } else {
2022                        Phase::Complete
2023                    };
2024                    if passed && self.state.config.merge.mode == MergeMode::Pr {
2025                        // Persist publication debt with the successful checkpoint.
2026                        self.state.pr_publication_pending = true;
2027                    } else if passed {
2028                        self.state.status = RunStatus::Ready;
2029                    }
2030                    self.state.save()?;
2031                    if !passed {
2032                        continue;
2033                    }
2034                    // Existing PR ownership and push lease stay in place. The
2035                    // normal publisher owes the new reviewed/gated SHA, and
2036                    // land's fresh owner approval remains mandatory.
2037                    if self.state.config.merge.mode != MergeMode::Pr {
2038                        self.state.status = RunStatus::Ready;
2039                        self.state.event(
2040                            "repair-grant",
2041                            "repair verified locally; grant carries no merge authorization",
2042                        );
2043                        self.state.save()?;
2044                        return Ok(());
2045                    }
2046                    return self.merge().await;
2047                }
2048                Phase::Complete => return Ok(()),
2049            }
2050        }
2051    }
2052
2053    async fn execute_graph(&mut self) -> Result<()> {
2054        // Even a terminal run may reenter through an explicit recovery action.
2055        // Restore before prep shortcuts, saved conversations or graph nodes use
2056        // a disposable path; GC may already have removed its original objects.
2057        let archive_home = crate::run::home().canonicalize()?;
2058        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
2059            let repo = self.state.repo.clone();
2060            let _coordination = self.coordinate(&repo).await?;
2061            ensure_setup_config(&mut self.state, &repo).await?;
2062            crate::retention::restore_execution_seats(
2063                &archive_home,
2064                &self.state,
2065                _coordination.as_ref(),
2066            )
2067            .await?;
2068        }
2069        self.state.configure_recovery_resume()?;
2070        if self
2071            .state
2072            .repair_grants
2073            .last()
2074            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
2075        {
2076            return self.execute_granted_repair().await;
2077        }
2078        if self
2079            .state
2080            .repair_grants
2081            .last()
2082            .is_some_and(|g| !g.succeeded)
2083        {
2084            self.state.status = RunStatus::Blocked;
2085            self.state.save()?;
2086            return Ok(());
2087        }
2088        // Preservation may invalidate an old empty classification. Resolve it
2089        // before any resume shortcut can reuse a verdict or publish a branch.
2090        self.persist_candidates().await?;
2091        // A saved base stop outranks stage markers, including legacy Voting.
2092        // Recovery never reenters implementation, deliberation or voting.
2093        if self.state.base_sync_blocked() {
2094            self.sync_to_base().await?;
2095            if !self.state.base_sync_blocked()
2096                && self.state.status != RunStatus::AlreadyInBase
2097                && !self.resource_halt()?
2098            {
2099                self.finish_after_tally().await?;
2100            }
2101            return Ok(());
2102        }
2103        if self
2104            .state
2105            .merge_decision
2106            .as_ref()
2107            .is_some_and(|decision| !decision.closed)
2108            && self.state.status != RunStatus::Landing
2109        {
2110            if crate::merge_decision::resumable_hold(&self.state) {
2111                self.state.merge = Some(MergeOutcome {
2112                    mode: MergeMode::Pr,
2113                    ok: false,
2114                    detail: self
2115                        .state
2116                        .merge_decision
2117                        .as_ref()
2118                        .unwrap()
2119                        .target
2120                        .pr
2121                        .clone(),
2122                    empty: false,
2123                });
2124                self.run_land().await?;
2125                self.settle_questions();
2126                return Ok(());
2127            }
2128            bail!(
2129                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2130            );
2131        }
2132        if self.state.config.graph.selected_implementer.is_some()
2133            && self.state.status == RunStatus::Blocked
2134            && self.state.tally.is_none()
2135            && self.state.candidate_preservation.is_empty()
2136            && self.state.interrupted_implementations.is_empty()
2137        {
2138            bail!(
2139                "selected implementation is held; explicitly release the owning task to retry it"
2140            );
2141        }
2142
2143        if self.state.merge_repairs.last().is_some_and(|r| {
2144            matches!(
2145                r.phase,
2146                crate::run::MergeRepairPhase::Requested
2147                    | crate::run::MergeRepairPhase::Fixing
2148                    | crate::run::MergeRepairPhase::Reviewing
2149            )
2150        }) {
2151            if self.repair_merge_review().await? {
2152                self.run_land().await?;
2153            }
2154            self.settle_questions();
2155            return Ok(());
2156        }
2157        // Publication already passed review and gate. Resume only that node:
2158        // syncing to a newer base here could change the verified commit.
2159        if self.state.pr_publication_pending {
2160            self.merge().await?;
2161            if self.resource_halt()? {
2162                return Ok(());
2163            }
2164            self.state.save()?;
2165            self.settle_questions();
2166            return Ok(());
2167        }
2168        // A run that already lost its quorum never resumes into the verdict
2169        // machinery: `deliberate` and `vote` would otherwise clobber the
2170        // stalled marker back to Voting and the run would keep going past a
2171        // verdict that is no longer trustworthy. Everything already recorded is
2172        // kept, so the run stays resumable (or foldable) for a human to pick up.
2173        //
2174        // On --resume the run gets one chance to repair itself: the seats a
2175        // rate limit took out are re-asked. If their quota has since reset and
2176        // the quorum is restored, the run picks up and finishes; otherwise it
2177        // stays stale and still-resumable for a later retry. If it does not
2178        // recover, the returned status stays `Stalled` and nothing was
2179        // clobbered (the recovery only mutates entries for the lost seats).
2180        if self.state.status == RunStatus::Stalled {
2181            if self.recover_stall().await? {
2182                self.finish_after_tally().await?;
2183            } else {
2184                // Still below quorum: persist the marker and stay resumable.
2185                self.state.save()?;
2186            }
2187            return Ok(());
2188        }
2189        // A run parked inside `land` - watching CI, mid fix-round, or
2190        // waiting on the owner's merge approval - resumes directly into it,
2191        // never back through `prep`. Everything before `merge` already
2192        // concluded; that is the only way `status` reaches `Landing` in the
2193        // first place. Re-walking `review_loop` first would also be actively
2194        // wrong: its own status recomputation (see its doc) treats any
2195        // clean round as reason to set `status` to `Gating`, which would
2196        // clobber this marker before `merge` ever ran, and this run would
2197        // never find its way back into `land` at all.
2198        if self.state.status == RunStatus::Landing
2199            || crate::handover::landing_checkpoint(&self.state)
2200        {
2201            self.state.land_sync_pending = true;
2202            self.state.save()?;
2203            self.run_land().await?;
2204            // `run_land` may have settled the run right here - CI came back
2205            // green and the PR merged, say - without ever passing back
2206            // through `merge`'s own trailing call. Whatever it left `status`
2207            // as is what this has to read.
2208            self.settle_questions();
2209            return Ok(());
2210        }
2211        // A run parked at `merge` on the owner's word about a withheld PR text
2212        // resumes straight into `merge`. Re-walking the tail would sync to the
2213        // base again (possibly rebasing, possibly spending the fixer) after the
2214        // gate had passed, and then push a tree nobody reviewed or gated.
2215        if self.state.github_text.is_some()
2216            && self.state.merge.is_none()
2217            && self.state.status == RunStatus::Gating
2218        {
2219            self.merge().await?;
2220            if self.resource_halt()? {
2221                return Ok(());
2222            }
2223            self.state.save()?;
2224            return Ok(());
2225        }
2226        if let Err(e) = self.prep().await {
2227            if e.is::<crate::repo_lock::ResourceBlocked>() {
2228                return Err(e);
2229            }
2230            self.state.status = RunStatus::Blocked;
2231            self.state
2232                .prep_failure
2233                .get_or_insert_with(|| format!("{e:#}"));
2234            self.state
2235                .event("prep", format!("preparation stopped: {e:#}"));
2236            self.state.save()?;
2237            return Err(e);
2238        }
2239        if self.park_here()? {
2240            return Ok(());
2241        }
2242        self.advise().await?;
2243        if self.park_here()? {
2244            return Ok(());
2245        }
2246        self.implement().await?;
2247        if self.park_here()? {
2248            return Ok(());
2249        }
2250        // `after_implement` already saved the state and settled any open
2251        // questions when it set this; nothing later in the graph has
2252        // anything to judge.
2253        if self.state.status == RunStatus::VerifiedNoop {
2254            return Ok(());
2255        }
2256        self.judge().await?;
2257        if self.park_here()? {
2258            return Ok(());
2259        }
2260        self.deliberate().await?;
2261        if self.park_here()? {
2262            return Ok(());
2263        }
2264        self.vote().await?;
2265        if self.park_here()? {
2266            return Ok(());
2267        }
2268        self.tally()?;
2269        // A verdict that lost its quorum is not trustworthy: do not review,
2270        // gate, or merge on it. Everything already done is kept, so the run
2271        // stays resumable (or foldable); the human can replace the agent that
2272        // ran out of quota and pick it up.
2273        if self.state.status == RunStatus::Stalled {
2274            // Persist the stalled marker now — the normal end-of-execute save
2275            // below is below this early return, and without it a resumed run
2276            // would reload a pre-tally status and keep going.
2277            self.state.save()?;
2278            return Ok(());
2279        }
2280        self.finish_after_tally().await?;
2281        Ok(())
2282    }
2283
2284    /// A wave could not start its agents because a shared resource stayed busy
2285    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2286    /// resumable. Unstarted seats have no result; completed peers retain
2287    /// their saved candidate state.
2288    fn stop_for_resource(&mut self) -> Result<()> {
2289        // Resource acquisition observes cancellation inside a node. Complete
2290        // the same durable park as every other node boundary before returning.
2291        if self.pause.parked() || self.interrupt.parked() {
2292            self.park_here()?;
2293            return Ok(());
2294        }
2295        if self.state.resource_block.is_some() {
2296            self.state.status = RunStatus::Blocked;
2297        }
2298        self.state.save()
2299    }
2300
2301    /// Stop walking the graph if a node ended on a resource block.
2302    fn resource_halt(&mut self) -> Result<bool> {
2303        if self.state.resource_block.is_none() {
2304            return Ok(false);
2305        }
2306        self.stop_for_resource()?;
2307        Ok(true)
2308    }
2309
2310    /// Park here if asked to, recording it in the run's own timeline.
2311    ///
2312    /// Returns whether the caller should stop walking the graph. The state is
2313    /// saved either way by the node that just finished; this adds the event so
2314    /// the operator's card says why a run that is neither finished nor moving
2315    /// is sitting where it is.
2316    fn park_here(&mut self) -> Result<bool> {
2317        let parking = self.pause.parked() || self.interrupt.parked();
2318        if !parking && self.state.resource_block.is_some() {
2319            self.stop_for_resource()?;
2320            return Ok(true);
2321        }
2322        // Either handle asking is enough - see `Pause`'s own doc for why
2323        // they are never the same one. `interrupt` is checked second so a
2324        // reason it carries is preferred in the message below over a plain
2325        // shutdown park racing it at the same boundary.
2326        if !self.pause.parked() && !self.interrupt.parked() {
2327            return Ok(false);
2328        }
2329        if self.state.parked {
2330            return Ok(true);
2331        }
2332        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2333        if let Some(block) = &mut self.state.resource_block
2334            && matches!(
2335                block.cause,
2336                crate::run::ResourceBlockCause::Cancelled { .. }
2337            )
2338        {
2339            block.cause = crate::run::ResourceBlockCause::Cancelled {
2340                park: cause.clone(),
2341            };
2342        }
2343        let why = match cause.as_ref().map(ParkCause::label) {
2344            Some(reason) => format!(
2345                "parked after `{}` ({reason}) — resume to carry on from here",
2346                self.state.status.as_str()
2347            ),
2348            None => format!(
2349                "parked after `{}` — resume to carry on from here",
2350                self.state.status.as_str()
2351            ),
2352        };
2353        self.state.event("park", why);
2354        self.state
2355            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2356        self.state.parked = true;
2357        self.state.save()?;
2358        Ok(true)
2359    }
2360
2361    /// Only a new explicit operator approval may retry failed preparation.
2362    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2363        if self.state.prep_failure.is_none() {
2364            return Ok(());
2365        }
2366        let failed_at = self
2367            .state
2368            .events
2369            .iter()
2370            .rev()
2371            .find(|event| event.node == "prep")
2372            .map_or(self.state.updated_at, |event| event.at);
2373        anyhow::ensure!(
2374            approval_at > failed_at,
2375            "preparation retry requires a new explicit release after the failure"
2376        );
2377        self.state.prep_failure = None;
2378        self.state.prep_complete = Some(false);
2379        self.state
2380            .event("prep-retry", "operator approved retry of preparation");
2381        Ok(())
2382    }
2383
2384    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2385    pub fn on_pause(&mut self, pause: Pause) {
2386        self.pause = pause;
2387    }
2388
2389    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2390    /// scheduler asking this one run - and no other - to park so a task
2391    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2392    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2393    /// handle.
2394    pub fn watch_interrupt(&mut self, pause: Pause) {
2395        self.interrupt = pause;
2396    }
2397
2398    /// Abandon this run's own open questions, once `status` has actually
2399    /// settled rather than merely paused.
2400    ///
2401    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2402    /// either back up with the candidates, the review round and the seat
2403    /// sessions already on disk, so a question an implementer asked mid-round
2404    /// may still get a real answer read by a real resume. Only the statuses
2405    /// `resumable` excludes are actually final: the run merged, it reached
2406    /// `Ready` with nothing left to do, it failed outright with no
2407    /// established point to continue from, or every candidate agreed, with
2408    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2409    /// every one of those the seat that asked is gone for good, exactly like
2410    /// the run being deleted under `magi run rm` - so the same cleanup
2411    /// applies, worded for what actually happened instead of "the run was
2412    /// deleted".
2413    ///
2414    /// Best-effort and silent on success: called from every place `status`
2415    /// can land on one of those three, including ones a resumed run revisits,
2416    /// so it must cost nothing when there was nothing open to begin with.
2417    fn settle_questions(&mut self) {
2418        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2419            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2420        }
2421    }
2422
2423    /// The tail of the graph after a trustworthy tally: fold losers, review,
2424    /// gate, merge, and persist.
2425    fn finish_after_tally(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
2426        Box::pin(self.finish_after_tally_inner())
2427    }
2428
2429    async fn finish_after_tally_inner(&mut self) -> Result<()> {
2430        // Before the losers are folded: their branches are what the graft
2431        // reads from.
2432        self.graft().await?;
2433        if self.resource_halt()? {
2434            return Ok(());
2435        }
2436        self.fold_losers().await?;
2437        // Before review starts, and again right before the gate: a run's
2438        // review rounds can themselves take long enough for the base to move
2439        // a second time, and the gate is the one node whose "green" gets
2440        // acted on.
2441        self.sync_to_base().await?;
2442        if self.hold_base_sync().await? || self.resource_halt()? {
2443            return Ok(());
2444        }
2445        if self.state.status == RunStatus::AlreadyInBase {
2446            return Ok(());
2447        }
2448        self.review_loop().await?;
2449        if self.resource_halt()? {
2450            return Ok(());
2451        }
2452        self.sync_to_base().await?;
2453        if self.hold_base_sync().await? || self.resource_halt()? {
2454            return Ok(());
2455        }
2456        if self.state.status == RunStatus::AlreadyInBase {
2457            return Ok(());
2458        }
2459        self.gate().await?;
2460        self.merge().await?;
2461        if self.resource_halt()? {
2462            return Ok(());
2463        }
2464        self.state.save()?;
2465        Ok(())
2466    }
2467
2468    // ---------------------------------------------------------------- prep
2469
2470    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2471    /// run with the step and its output in the event; it is never swallowed.
2472    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2473    /// resume starts over instead of finding candidates and missing seats.
2474    async fn setup_seat_worktree(
2475        &mut self,
2476        repo: &Path,
2477        wt: &Path,
2478        in_prep: bool,
2479        coordination: Option<&crate::repo_lock::SeatLease>,
2480    ) -> Result<()> {
2481        let result = match ensure_setup_config(&mut self.state, repo).await {
2482            Ok(()) => {
2483                worktree_setup::prepare_with(
2484                    &self.state.config,
2485                    repo,
2486                    wt,
2487                    crate::build_target::Coordination::Held(coordination),
2488                )
2489                .await
2490            }
2491            Err(e) => Err(e),
2492        };
2493        if let Err(e) = result {
2494            git::worktree_remove_clean(repo, wt).await.ok();
2495            if in_prep {
2496                self.rollback_prep(repo).await;
2497            }
2498            self.state.status = RunStatus::Blocked;
2499            self.state.event(
2500                "setup",
2501                format!("worktree setup failed in {}: {e:#}", wt.display()),
2502            );
2503            self.state.save()?;
2504            return Err(e);
2505        }
2506        Ok(())
2507    }
2508
2509    /// Undo everything `prep` created, so a resume re-runs it whole.
2510    async fn rollback_prep(&mut self, repo: &Path) {
2511        let root = self.state.worktree_root();
2512        for c in std::mem::take(&mut self.state.candidates) {
2513            if matches!(
2514                git::worktree_remove_clean(repo, &c.worktree).await,
2515                Ok(true)
2516            ) {
2517                git::branch_delete(repo, &c.branch).await.ok();
2518            } else {
2519                self.state.candidates.push(c);
2520            }
2521        }
2522        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2523        for j in 1..=seats {
2524            remove_read_seat_worktree(&mut self.state, &root.join(format!("judge-{j}")))
2525                .await
2526                .ok();
2527        }
2528        for k in 1..=self.state.config.graph.advisors {
2529            remove_read_seat_worktree(&mut self.state, &root.join(format!("advisor-{k}")))
2530                .await
2531                .ok();
2532        }
2533    }
2534
2535    async fn prep(&mut self) -> Result<()> {
2536        if let Some(reason) = &self.state.prep_failure {
2537            bail!(
2538                "candidate preparation previously failed: {reason}; correct the source before retrying"
2539            );
2540        }
2541        if self
2542            .state
2543            .prep_complete
2544            .unwrap_or(!self.state.candidates.is_empty())
2545        {
2546            return Ok(());
2547        }
2548        let repo = self.state.repo.clone();
2549        // Includes rollback, config, refs, checkout creation and setup. No
2550        // repository mutation may precede the real shared lock acquisition.
2551        let _coordination = self.coordinate(&repo).await?;
2552        let base = self.state.base_commit.clone();
2553        let plan = self.state.sources.plan(&repo, &base).await?;
2554        self.rollback_prep(&repo).await;
2555        if !self.state.candidates.is_empty() {
2556            bail!(
2557                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2558            );
2559        }
2560        self.state.prep_complete = Some(false);
2561        self.state.status = RunStatus::Prep;
2562        self.state.save()?;
2563        let repo = self.state.repo.clone();
2564        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2565        let root = self.state.worktree_root();
2566        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2567
2568        // The hook is the write-time half of the blindness contract; the
2569        // presentation filter in `blind` is the half that cannot be bypassed.
2570        let hooks_dir = self.state.dir().join("hooks");
2571        if self.state.config.blind.commit_msg_hook {
2572            std::fs::create_dir_all(&hooks_dir)
2573                .with_context(|| format!("create {}", hooks_dir.display()))?;
2574            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2575            let path = hooks_dir.join("commit-msg");
2576            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2577            make_executable(&path)?;
2578            // Ref-counted rather than a plain idempotent set: with more than
2579            // one run able to be in flight in the same repository at once
2580            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2581            // true?" check cannot tell "another run of mine still needs
2582            // this" from "nobody does", and the run that happens to finish
2583            // first would disable the hook out from under a sibling still
2584            // relying on it.
2585            git::acquire_worktree_config(&repo).await?;
2586            self.state.enabled_worktree_config = true;
2587        }
2588
2589        // `[worktree] setup` hides its products per worktree, which needs the
2590        // same per-worktree config the hook does. Held until fold, like it.
2591        ensure_setup_config(&mut self.state, &repo).await?;
2592
2593        for (index, (spec, label)) in self
2594            .roles
2595            .implementers
2596            .clone()
2597            .into_iter()
2598            .zip(labels)
2599            .enumerate()
2600        {
2601            let branch = self.state.branch_for(label);
2602            let worktree = root.join(format!("cand-{label}"));
2603            // Persist intent before Git can leave a partially created tree.
2604            // Candidate registration follows setup, so it cannot inventory
2605            // crashes or refusals during creation itself.
2606            self.state.event(
2607                "prep-intent",
2608                serde_json::json!({
2609                    "branch": branch,
2610                    "worktree": worktree,
2611                    "start": start,
2612                    "picks": plan.picks,
2613                })
2614                .to_string(),
2615            );
2616            self.state.save()?;
2617            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2618            if self.state.config.blind.commit_msg_hook {
2619                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2620            }
2621            git::local_exclude(&worktree, "/.magi/").await?;
2622            if let Err(e) = worktree_setup::prepare_with(
2623                &self.state.config,
2624                &repo,
2625                &worktree,
2626                crate::build_target::Coordination::Held(_coordination.as_ref()),
2627            )
2628            .await
2629            {
2630                // Nothing of this prep may survive: a resume must start over,
2631                // not find half the candidates and skip the rest.
2632                git::worktree_remove_clean(&repo, &worktree).await.ok();
2633                git::branch_delete(&repo, &branch).await.ok();
2634                self.rollback_prep(&repo).await;
2635                self.state.status = RunStatus::Blocked;
2636                self.state
2637                    .event("prep", format!("worktree setup failed: {e:#}"));
2638                if !e.is::<crate::repo_lock::ResourceBlocked>() {
2639                    self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2640                }
2641                self.state.save()?;
2642                return Err(e);
2643            }
2644            self.state.candidates.push(Candidate {
2645                index,
2646                label,
2647                agent: spec.id.clone(),
2648                branch,
2649                worktree,
2650                summary: String::new(),
2651                stat: String::new(),
2652                files: 0,
2653                commits: 0,
2654                empty: false,
2655                failed: None,
2656                verified_noop: None,
2657                duration_ms: 0,
2658                folded: false,
2659            });
2660            self.state.save()?;
2661            for pick in &plan.picks {
2662                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2663                {
2664                    self.state.status = RunStatus::Blocked;
2665                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2666                    self.state
2667                        .event("prep", format!("cannot apply referenced commit: {e}"));
2668                    self.state.save()?;
2669                    return Err(e);
2670                }
2671            }
2672        }
2673
2674        for j in 1..=self.roles.judges.len() {
2675            let wt = root.join(format!("judge-{j}"));
2676            if !wt.exists() {
2677                git::worktree_add_detached(&repo, &wt, &base).await?;
2678                self.setup_seat_worktree(&repo, &wt, true, _coordination.as_ref())
2679                    .await?;
2680            }
2681        }
2682
2683        // Disposable, detached checkouts for the design-deliberation stage's
2684        // advisor seats — the same shape as the judges' above, at the same
2685        // base commit, since advisors also only ever read. Sized off the
2686        // configured count directly rather than a resolved roster: unlike
2687        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2688        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2689        // `prep` has no `ResolvedRoles` field to read a count from here.
2690        if self.state.config.graph.advise {
2691            for k in 1..=self.state.config.graph.advisors {
2692                let wt = root.join(format!("advisor-{k}"));
2693                if !wt.exists() {
2694                    git::worktree_add_detached(&repo, &wt, &base).await?;
2695                    self.setup_seat_worktree(&repo, &wt, true, _coordination.as_ref())
2696                        .await?;
2697                }
2698            }
2699        }
2700
2701        // A judge cannot tell it is looking at its own patch — the seats keep
2702        // separate conversations — but a panel that shares agents with the
2703        // field is less independent than it looks, and that is worth saying out
2704        // loud once per run rather than leaving it in the config.
2705        let authors: Vec<&str> = self
2706            .roles
2707            .implementers
2708            .iter()
2709            .map(|a| a.id.as_str())
2710            .collect();
2711        let overlap: Vec<String> = self
2712            .roles
2713            .judges
2714            .iter()
2715            .enumerate()
2716            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2717            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2718            .collect();
2719        if !overlap.is_empty() {
2720            let note = format!(
2721                "{} also authored a candidate; blind, but the panel is less \
2722                 independent than {} distinct agents would be",
2723                overlap.join(", "),
2724                self.roles.judges.len()
2725            );
2726            self.state.event("prep", note);
2727        }
2728
2729        self.state.event(
2730            "prep",
2731            format!(
2732                "{} candidates, {} judges, base {} ({})",
2733                self.state.candidates.len(),
2734                self.roles.judges.len(),
2735                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2736                self.state.base_branch
2737            ),
2738        );
2739        self.state.prep_complete = Some(true);
2740        self.state.status = RunStatus::Implementing;
2741        self.state.save()?;
2742        Ok(())
2743    }
2744
2745    // -------------------------------------------------------------- advise
2746
2747    /// The design-deliberation stage: independent, read-only advisor seats
2748    /// each sketch a design before any implementer touches the repository,
2749    /// and (when at least one produced a usable proposal) a synthesis seat
2750    /// blends them into a brief `implement` carries in every candidate's
2751    /// prompt.
2752    ///
2753    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2754    /// advisors` is the proposal count. Everything here is best-effort and
2755    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2756    /// that cannot reach quota, or a synthesis seat that produced nothing
2757    /// usable all leave `implement` exactly as it was before this stage
2758    /// existed — the task instruction alone — rather than failing the whole
2759    /// competition over an enrichment stage. Every outcome is still recorded
2760    /// as an event, so a run that got nothing from this stage says why.
2761    ///
2762    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2763    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2764    /// resumed run whose stage failed would re-run it, and re-spend the
2765    /// agent calls, on every reentry before `implement`.
2766    ///
2767    /// Also skipped once any candidate shows implementation progress — the
2768    /// exact predicate `implement` itself uses to decide a candidate is no
2769    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2770    /// is not enough: a run created by an older binary that predates this
2771    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2772    /// an already-`Implementing`-or-later run under this build would
2773    /// otherwise walk straight back through `prep` (a no-op once candidates
2774    /// exist) into this node and spawn every advisor seat against worktrees
2775    /// `prep` never recreated — after implementation has already started,
2776    /// which is exactly the invariant this stage exists to guarantee.
2777    async fn advise(&mut self) -> Result<()> {
2778        let implement_untouched = self
2779            .state
2780            .candidates
2781            .iter()
2782            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2783        if !self.state.config.graph.advise || self.state.advise_attempted {
2784            return Ok(());
2785        }
2786        if !implement_untouched {
2787            self.state.event(
2788                "advise",
2789                "skipping the design-deliberation stage: at least one \
2790                 candidate already shows implementation progress, so this \
2791                 run is past the point the stage exists to run before"
2792                    .to_owned(),
2793            );
2794            self.state.advise_attempted = true;
2795            self.state.save()?;
2796            return Ok(());
2797        }
2798        let run_id = self.state.id.clone();
2799        let prompts = self.state.config.prompts.clone();
2800        let instruction = self.state.instruction.clone();
2801        let language = self.state.config.graph.language.clone();
2802        let root = self.state.worktree_root();
2803        let n = self.state.config.graph.advisors;
2804        let where_recorded = self.state.dir().join("run.json");
2805
2806        let seats = match self.state.config.advisors() {
2807            Ok(seats) if !seats.is_empty() => seats,
2808            Ok(_) => {
2809                self.state.event(
2810                    "advise",
2811                    format!(
2812                        "[graph] advisors is 0; skipping the design-deliberation \
2813                         stage and continuing without a synthesis brief (see {})",
2814                        where_recorded.display()
2815                    ),
2816                );
2817                self.state.advise_attempted = true;
2818                self.state.save()?;
2819                return Ok(());
2820            }
2821            Err(e) => {
2822                self.state.event(
2823                    "advise",
2824                    format!(
2825                        "could not resolve advisor seats ({e:#}); continuing \
2826                         without a design-deliberation brief (see {})",
2827                        where_recorded.display()
2828                    ),
2829                );
2830                self.state.advise_attempted = true;
2831                self.state.save()?;
2832                return Ok(());
2833            }
2834        };
2835
2836        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2837        let artifacts = agent::artifacts_dir(&self.state.dir());
2838        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2839
2840        let mut jobs = Vec::new();
2841        for (i, spec) in seats.iter().cloned().enumerate() {
2842            let seat_key = format!("advisor-{}", i + 1);
2843            let seat = self.seat(&seat_key, &spec.id);
2844            jobs.push(SeatJob {
2845                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2846                spec,
2847                seat,
2848                cwd: worktrees[i % worktrees.len()].clone(),
2849                timeout,
2850                allow_write: false,
2851                sessions: false,
2852                artifacts: artifacts.clone(),
2853                stem: seat_key,
2854                handover: None,
2855            });
2856        }
2857
2858        self.state.event(
2859            "advise",
2860            format!(
2861                "{} advisor seat(s) sketching a design in parallel",
2862                jobs.len()
2863            ),
2864        );
2865        let mut quota_losses = Vec::new();
2866        let cache = self.state.config.cache_dir();
2867        let ctx = WaveCtx {
2868            carry_seats: false,
2869            pauses: Some((&self.pause, &self.interrupt)),
2870            run: &run_id,
2871            node: "advise",
2872            prompts: &prompts,
2873            cache: cache.as_deref(),
2874            round: None,
2875        };
2876        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2877        let results = ask_json_wave::<Proposal>(
2878            jobs,
2879            Arc::clone(&self.sem),
2880            self.state.config.graph.retries,
2881            &advisor_roster,
2882            &ctx,
2883            &mut quota_losses,
2884            &mut self.state,
2885            &|p: &Proposal| p.validate(),
2886        )
2887        .await;
2888        if self.state.resource_block.is_some() {
2889            return self.stop_for_resource();
2890        }
2891        self.state.quota.extend(quota_losses);
2892
2893        let mut records = Vec::with_capacity(results.len());
2894        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2895            let agent_id = seat.agent.clone();
2896            self.state.seats.insert(seat.key.clone(), seat);
2897            match res {
2898                Ok((proposal, out)) => {
2899                    self.state
2900                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2901                    records.push(advise::AdvisorRecord::proposed(
2902                        i + 1,
2903                        agent_id,
2904                        proposal,
2905                        out.duration_ms,
2906                    ));
2907                }
2908                Err(e) => {
2909                    self.state.event(
2910                        "advise",
2911                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2912                    );
2913                    records.push(advise::AdvisorRecord::failed(
2914                        i + 1,
2915                        agent_id,
2916                        e.to_string(),
2917                    ));
2918                }
2919            }
2920        }
2921
2922        let mut advice = advise::Advice {
2923            records,
2924            synthesis: None,
2925        };
2926        if advice.proposals().is_empty() {
2927            self.state.event(
2928                "advise",
2929                "no advisor produced a usable proposal; continuing without a \
2930                 synthesis brief"
2931                    .to_owned(),
2932            );
2933        } else {
2934            match self
2935                .synthesize_brief(
2936                    &advice,
2937                    &instruction,
2938                    &language,
2939                    &worktrees[0],
2940                    &artifacts,
2941                    &run_id,
2942                    &prompts,
2943                    cache.as_deref(),
2944                )
2945                .await
2946            {
2947                Ok(Some(text)) => {
2948                    self.state.event(
2949                        "advise",
2950                        "synthesized a design brief for the implementer".to_owned(),
2951                    );
2952                    advice.synthesis = Some(text);
2953                }
2954                Ok(None) => {
2955                    self.state.event(
2956                        "advise",
2957                        "the synthesis seat produced nothing usable; continuing \
2958                         without a design brief"
2959                            .to_owned(),
2960                    );
2961                }
2962                Err(e) => {
2963                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2964                        return Err(e);
2965                    }
2966                    self.state.event(
2967                        "advise",
2968                        format!("could not synthesize a design brief: {e:#}"),
2969                    );
2970                }
2971            }
2972        }
2973        advise::apply_reflection(&mut advice);
2974
2975        self.state.advice = Some(advice);
2976        self.state.advise_attempted = true;
2977        self.state.save()?;
2978        Ok(())
2979    }
2980
2981    /// The synthesis seat: reads every advisor's proposal and blends them
2982    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2983    /// out of [`Runner::advise`] only for readability — it is not called
2984    /// anywhere else.
2985    ///
2986    /// Picked the same way [`crate::talk`]'s standing conversation and
2987    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2988    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2989    /// order (a claude seat, else the first runnable agent in roster order)
2990    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2991    /// in [`crate::config`] for why a dedicated field exists here at all.
2992    #[allow(clippy::too_many_arguments)]
2993    async fn synthesize_brief(
2994        &mut self,
2995        advice: &advise::Advice,
2996        instruction: &str,
2997        language: &str,
2998        cwd: &Path,
2999        artifacts: &Path,
3000        run_id: &str,
3001        prompts: &Prompts,
3002        cache: Option<&Path>,
3003    ) -> Result<Option<String>> {
3004        let chain = agent::pick_chain(
3005            &self.state.config.agents,
3006            self.state.config.roles.synthesizer.as_ref(),
3007            &agent::installed,
3008            "synthesizer",
3009        )?;
3010        let proposals = advice.proposals();
3011        let mut prompt = prompt::with_overlay(
3012            prompt::synthesize_brief(instruction, &proposals, language),
3013            prompts.overlay("advise"),
3014        );
3015        if cache.is_some() {
3016            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
3017            // below — see `prompt::build_cache_note`'s doc for why telling a
3018            // read-only seat to build through the shared cache is exactly how
3019            // a sandbox's write refusal gets misread as a defect.
3020            prompt.push('\n');
3021            prompt.push_str(&prompt::build_cache_note("advise", false));
3022        }
3023        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
3024        // Each id is tried once, in order; a quota hit, error or unusable
3025        // answer moves to the next. The seat is single-turn (`sessions:
3026        // false`) and the prompt is the whole context, so a fallback agent
3027        // needs nothing carried over.
3028        let mut last = None;
3029        for (n, spec) in chain.iter().enumerate() {
3030            if n > 0 {
3031                self.state
3032                    .event("advise", format!("synthesis falling back to {}", spec.id));
3033            }
3034            let mut seat = self.seat("advise-synthesis", &spec.id);
3035            let wait = self.repository_wait();
3036            let outcome = wait
3037                .scope(agent::invoke(
3038                    spec,
3039                    &mut seat,
3040                    &Invocation {
3041                        coordination: self.state.config.coordination.as_ref(),
3042                        fetched_base: None,
3043                        cwd,
3044                        prompt: &prompt,
3045                        timeout,
3046                        allow_write: false,
3047                        sessions: false,
3048                        artifacts,
3049                        stem: &if n == 0 {
3050                            "advise-synthesis".to_owned()
3051                        } else {
3052                            format!("advise-synthesis-{}", spec.id)
3053                        },
3054                        run: run_id,
3055                        node: "advise",
3056                        cache_dir: None,
3057                        attachments: &[],
3058                        writable: &[],
3059                    },
3060                ))
3061                .await;
3062            if outcome.is_ok() {
3063                self.state.seats.insert(seat.key.clone(), seat);
3064            }
3065            let advance = agent::chain_advances(&outcome);
3066            last = Some(outcome);
3067            if !advance {
3068                break;
3069            }
3070        }
3071        // Exhausted: the last attempt's result is what a single failed seat
3072        // would have produced.
3073        let out = last.expect("a chain holds at least one agent")?;
3074        if !out.usable() {
3075            return Ok(None);
3076        }
3077        let text =
3078            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
3079        Ok((!text.trim().is_empty()).then_some(text))
3080    }
3081
3082    // ----------------------------------------------------------- implement
3083
3084    async fn implement(&mut self) -> Result<()> {
3085        self.persist_candidates().await?;
3086        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3087        // agent files with `magi task add` name the run that paid for it. The
3088        // prompt overlay is cloned alongside it because the waves borrow it
3089        // while `self` is mutably borrowed by the node's own bookkeeping.
3090        let run_id = self.state.id.clone();
3091        let prompts = self.state.config.prompts.clone();
3092        let todo: Vec<usize> = self
3093            .state
3094            .candidates
3095            .iter()
3096            .enumerate()
3097            .filter(|(_, c)| {
3098                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
3099                    && c.failed.is_none()
3100                    && !c.empty
3101            })
3102            .map(|(i, _)| i)
3103            .collect();
3104        if todo.is_empty() {
3105            return self.after_implement();
3106        }
3107        self.state.status = RunStatus::Implementing;
3108
3109        let language = self.state.config.graph.language.clone();
3110        let github_language = self.state.config.graph.github_language.clone();
3111        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3112        let sessions = self.state.config.graph.sessions;
3113        let artifacts = agent::artifacts_dir(&self.state.dir());
3114        // The design-deliberation stage's blended brief, when `advise` found
3115        // one — carried into every implementer's prompt the same way
3116        // regardless of which candidate it is.
3117        let brief = self
3118            .state
3119            .advice
3120            .as_ref()
3121            .and_then(|a| a.synthesis.as_deref())
3122            .map(str::to_owned);
3123        let attachments = self.state.attachments.clone();
3124
3125        let mut jobs = Vec::new();
3126        for &i in &todo {
3127            let (index, label, worktree) = {
3128                let c = &self.state.candidates[i];
3129                (c.index, c.label, c.worktree.clone())
3130            };
3131            let spec = self.roles.implementers[index].clone();
3132            let seat_key = format!("impl-{label}");
3133            let seat = self.seat(&seat_key, &spec.id);
3134            let instruction = seeded_instruction(&self.state);
3135            jobs.push(SeatJob {
3136                spec,
3137                seat,
3138                prompt: prompt::implement_in(
3139                    &instruction,
3140                    &worktree.to_string_lossy(),
3141                    &language,
3142                    &github_language,
3143                    brief.as_deref(),
3144                    &attachments,
3145                ),
3146                cwd: worktree,
3147                timeout,
3148                allow_write: true,
3149                sessions,
3150                artifacts: artifacts.clone(),
3151                stem: format!("impl-{label}"),
3152                handover: None,
3153            });
3154        }
3155
3156        self.state.event(
3157            "implement",
3158            format!("{} candidates in parallel", jobs.len()),
3159        );
3160        // Kept so a seat whose CLI hung up can be asked again from the same
3161        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3162        // can update a seat's own entry once a fallback agent takes it over —
3163        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3164        // whichever agent actually answered, not the one that quota'd out.
3165        let mut sent = jobs.clone();
3166        let cache = self.state.config.cache_dir();
3167        let ctx = WaveCtx {
3168            carry_seats: false,
3169            pauses: Some((&self.pause, &self.interrupt)),
3170            run: &run_id,
3171            node: "implement",
3172            prompts: &prompts,
3173            cache: cache.as_deref(),
3174            round: None,
3175        };
3176        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3177        if !wave_blocked(&results) {
3178            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3179                .await;
3180            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3181                .await?;
3182            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3183                .await;
3184        }
3185        // Do not finalize failed candidates when the entire wave needs machine
3186        // recovery: leaving them pending resumes this node with its work intact.
3187        if !results.is_empty()
3188            && results
3189                .iter()
3190                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3191        {
3192            let losses: Vec<_> = results
3193                .iter()
3194                .map(|(_, seat, out)| {
3195                    let AgentOutcome::Quota(o) = out else {
3196                        unreachable!()
3197                    };
3198                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3199                })
3200                .collect();
3201            self.state.quota.extend(losses.clone());
3202            for (_, seat, _) in results {
3203                self.state.seats.insert(seat.key.clone(), seat);
3204            }
3205            for &i in &todo {
3206                let label = self.state.candidates[i].label;
3207                self.state.interrupted_implementations.insert(label);
3208                self.state
3209                    .candidate_preservation
3210                    .insert(label, "machine outage awaiting persistence".into());
3211            }
3212            // Preserve partial edits too, while keeping every interrupted seat
3213            // pending even when its tree already contains an inherited patch.
3214            block_machine_outage(&mut self.state, "implement", &losses);
3215            self.state.save()?;
3216            self.persist_candidates().await?;
3217            for &i in &todo {
3218                self.state
3219                    .interrupted_implementations
3220                    .insert(self.state.candidates[i].label);
3221            }
3222            return self.stop_for_resource();
3223        }
3224        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3225
3226        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3227            && results
3228                .iter()
3229                .any(|(_, _, out)| FailClass::of(out).is_some());
3230        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3231            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3232                // Executed turns interrupted by disk pressure already carry
3233                // a preservation barrier from wave. Admission-only waits do
3234                // not. Keep the interrupted seat's session for recovery too.
3235                if self
3236                    .state
3237                    .candidate_preservation
3238                    .contains_key(&self.state.candidates[i].label)
3239                {
3240                    self.state.seats.insert(seat.key.clone(), seat);
3241                }
3242                continue;
3243            }
3244            let seat_key = seat.key.clone();
3245            // A quota fallback (`resume_seat_handovers`) may have handed this
3246            // seat to a different agent than the one `prep` recorded on the
3247            // candidate; the stats tables and any later fixer-defaults-to-
3248            // winner's-author lookup must credit whoever actually answered —
3249            // unless every fallback also quota'd out, in which case nobody
3250            // actually answered and crediting the last agent tried would
3251            // erase every earlier agent's own quota loss from the stats
3252            // tables instead of just this one seat's.
3253            let agent = seat.agent.clone();
3254            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3255            self.state.seats.insert(seat.key.clone(), seat);
3256            let label = self.state.candidates[i].label;
3257            self.state.interrupted_implementations.remove(&label);
3258
3259            let (summary, duration, failed, verified_claim) = match out {
3260                AgentOutcome::Ok(o) => {
3261                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3262                    let failed = (!o.usable()).then(|| {
3263                        if o.timed_out {
3264                            "agent timed out".to_owned()
3265                        } else {
3266                            format!(
3267                                "agent exited with {}",
3268                                crate::notices::exit_status(o.exit_code)
3269                            )
3270                        }
3271                    });
3272                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3273                    (text, o.duration_ms, failed, verified_claim)
3274                }
3275                // Left un-resumed by `resume_undelivered` (a dirty tree
3276                // already rescues the work, or there was no session left to
3277                // resume into) — reported like the ordinary failure it is,
3278                // never as if `o.text` (the CLI's raw error JSON) were an
3279                // answer.
3280                AgentOutcome::Dropped(o) => {
3281                    let why = o
3282                        .dropped
3283                        .as_ref()
3284                        .map(|d| d.why.as_str())
3285                        .unwrap_or("the CLI ended the stream without delivering its answer");
3286                    (
3287                        String::new(),
3288                        o.duration_ms,
3289                        Some(format!("the CLI dropped the stream ({why})")),
3290                        None,
3291                    )
3292                }
3293                AgentOutcome::Quota(o) => {
3294                    self.state.quota.push(QuotaLoss::from_report(
3295                        seat_key,
3296                        "implement",
3297                        o.quota.as_ref(),
3298                    ));
3299                    (
3300                        String::new(),
3301                        o.duration_ms,
3302                        Some("rate limited (quota); produced no change".to_owned()),
3303                        None,
3304                    )
3305                }
3306                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3307                    (String::new(), 0, Some(e), None)
3308                }
3309            };
3310
3311            // Save every completed turn before any fallible Git operation. A failed
3312            // rescue must not erase the reply or prevent collection of other seats.
3313            let c = &mut self.state.candidates[i];
3314            if !exhausted_the_fallback_chain {
3315                c.agent = agent;
3316            }
3317            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3318            c.duration_ms = duration;
3319            c.failed = failed;
3320            c.verified_noop = verified_claim;
3321            self.state
3322                .candidate_preservation
3323                .insert(label, "awaiting persistence".to_owned());
3324        }
3325        self.state.save()?;
3326        self.persist_candidates().await?;
3327
3328        if resource_blocked {
3329            return self.stop_for_resource();
3330        }
3331        if selected_failed {
3332            self.state.status = RunStatus::Blocked;
3333            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3334            self.state.save()?;
3335            bail!(
3336                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3337            );
3338        }
3339        self.after_implement()
3340    }
3341
3342    /// Retry only persistence after a completed implementation turn.
3343    async fn persist_candidates(&mut self) -> Result<()> {
3344        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3345        for label in labels {
3346            let i = self
3347                .state
3348                .candidates
3349                .iter()
3350                .position(|c| c.label == label)
3351                .context("pending candidate is missing")?;
3352            let c = self.state.candidates[i].clone();
3353            let interrupted = self.state.interrupted_implementations.contains(&label);
3354            let result = async {
3355                let wait = crate::repo_lock::WaitPolicy::new(
3356                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3357                    || false,
3358                );
3359                let _coordination =
3360                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3361                // Best effort evidence requires no index writes. A successful snapshot
3362                // never authorizes removal of a tree whose rescue failed.
3363                if let Err(e) = git::snapshot_uncommitted(
3364                    &c.worktree,
3365                    &self
3366                        .state
3367                        .dir()
3368                        .join("artifacts")
3369                        .join(format!("candidate-{label}-recovery")),
3370                )
3371                .await
3372                {
3373                    self.state.event(
3374                        "preserve",
3375                        format!("candidate {label} snapshot incomplete: {e:#}"),
3376                    );
3377                }
3378                let rescued = git::rescue_commit(
3379                    &c.worktree,
3380                    &format!("magi: candidate {label} (uncommitted work)"),
3381                )
3382                .await?;
3383                self.state.note_withheld("implement", &rescued.withheld);
3384                let base = &self.state.base_commit;
3385                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3386                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3387                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3388                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3389                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3390                if c.empty && commits > 0 && !patch.trim().is_empty()
3391                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3392                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3393                        || self.state.tally.is_some())
3394                {
3395                    // Keep the old classification and pending marker so every
3396                    // retry stops too. Retain decisions as recovery evidence;
3397                    // only a supported recovery operation may replace them.
3398                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3399                }
3400                let c = &mut self.state.candidates[i];
3401                c.commits = commits;
3402                c.stat = stat;
3403                c.files = files;
3404                let no_changes = commits == 0 || patch.trim().is_empty();
3405                c.empty = no_changes && !interrupted;
3406                if interrupted && no_changes {
3407                    c.failed = None;
3408                    c.verified_noop = None;
3409                }
3410                if !c.empty {
3411                    c.failed = None;
3412                    c.verified_noop = None;
3413                }
3414                self.state.event(
3415                    "preserve",
3416                    format!("candidate {label}: {files} files, {commits} commits"),
3417                );
3418                Ok::<_, anyhow::Error>(())
3419            }
3420            .await;
3421            match result {
3422                Ok(()) => {
3423                    self.state.candidate_preservation.remove(&label);
3424                    if !interrupted || self.state.candidates[i].files > 0 {
3425                        self.state.interrupted_implementations.remove(&label);
3426                    }
3427                }
3428                Err(e) => {
3429                    let reason = format!("candidate {label} preservation failed: {e:#}");
3430                    self.state.event("preserve", &reason);
3431                    self.state.candidate_preservation.insert(label, reason);
3432                }
3433            }
3434            if self.state.candidate_preservation.is_empty()
3435                && self.state.status == RunStatus::Blocked
3436                && self.state.candidates.iter().all(|c| c.failed.is_none())
3437            {
3438                // Persist the recovered phase with the cleared pending marker;
3439                // a crash between saves must not turn this into a selected-seat
3440                // failure hold on the next resume.
3441                self.state.status = RunStatus::Implementing;
3442                self.state.event(
3443                    "preserve",
3444                    "candidate persistence recovered; implementation can continue",
3445                );
3446            }
3447            self.state.save()?;
3448        }
3449        if !self.state.candidate_preservation.is_empty() {
3450            self.state.status = RunStatus::Blocked;
3451            self.state.save()?;
3452            bail!(
3453                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3454            );
3455        }
3456        Ok(())
3457    }
3458
3459    /// Ask again, once, for work a CLI did and then failed to hand over.
3460    ///
3461    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3462    /// status with an empty response and a usage report showing output tokens,
3463    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3464    /// seven minutes and 14,267 output tokens that arrived as an empty
3465    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3466    ///
3467    /// Two conditions, and both matter:
3468    ///
3469    /// - **Only when the tree is untouched.** Often the agent has already
3470    ///   written its files and only the closing message was lost; the rescue
3471    ///   commit below picks that up and there is nothing to ask for. Re-asking
3472    ///   then would pay for a second implementation of work already on disk.
3473    /// - **Once.** A CLI that drops one stream can drop the next, and this
3474    ///   node is the most expensive in the graph.
3475    ///
3476    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3477    /// dropped reply still carried its `conversation_id`, so the seat is asked
3478    /// to finish what it was doing rather than sent the whole task again. It
3479    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3480    /// node's - for the same reason a re-ranked judge does: restating finished
3481    /// work is not the work.
3482    ///
3483    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3484    /// same way until it resets, while an abandoned conversation is still
3485    /// there to be picked up.
3486    async fn resume_undelivered(
3487        &mut self,
3488        results: &mut [(usize, SeatState, AgentOutcome)],
3489        sent: &[SeatJob],
3490        prompts: &Prompts,
3491        run_id: &str,
3492    ) {
3493        for (wi, seat, out) in results.iter_mut() {
3494            let Some(dropped) = (match &*out {
3495                AgentOutcome::Dropped(o) => o.dropped.clone(),
3496                _ => None,
3497            }) else {
3498                continue;
3499            };
3500            let Some(job) = sent.get(*wi) else { continue };
3501            // Already on disk? Then only the closing message was lost.
3502            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3503                self.state.event(
3504                    "implement",
3505                    format!(
3506                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3507                         work is in the tree",
3508                        seat.key, dropped.output_tokens, dropped.why
3509                    ),
3510                );
3511                continue;
3512            }
3513            // The re-ask only makes sense as a resume: `resume_after_drop`
3514            // says nothing about the task, trusting the seat to still hold it.
3515            // Without a session to resume — sessions disabled, or this CLI's
3516            // drop shape happened not to carry a session id — that prompt
3517            // would open a brand-new conversation with no context at all,
3518            // which is worse than leaving this as the ordinary failure it
3519            // already is.
3520            if !has_context(&job.spec, seat, job.sessions) {
3521                self.state.event(
3522                    "implement",
3523                    format!(
3524                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3525                         is no session left to resume",
3526                        seat.key, dropped.output_tokens, dropped.why
3527                    ),
3528                );
3529                continue;
3530            }
3531            self.state.event(
3532                "implement",
3533                format!(
3534                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3535                     conversation",
3536                    seat.key, dropped.output_tokens, dropped.why
3537                ),
3538            );
3539            let mut retry = job.clone();
3540            retry.seat = seat.clone();
3541            retry.prompt = prompt::resume_after_drop(&dropped.why);
3542            retry.timeout = retry_budget(job.timeout, true);
3543            retry.stem = format!("{}-resume", job.stem);
3544            let cache = self.state.config.cache_dir();
3545            let ctx = WaveCtx {
3546                carry_seats: false,
3547                pauses: Some((&self.pause, &self.interrupt)),
3548                run: run_id,
3549                node: "implement",
3550                prompts,
3551                cache: cache.as_deref(),
3552                round: None,
3553            };
3554            let (resumed_seat, resumed) =
3555                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3556            *seat = resumed_seat;
3557            *out = resumed;
3558        }
3559    }
3560
3561    /// Fall an implement seat through to the next untried agent in the
3562    /// implementer roster when it lost to quota — or, since the handover was
3563    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3564    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3565    /// quota path itself being unchanged — instead of leaving the
3566    /// seat's loss final the moment one agent's account runs dry.
3567    ///
3568    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3569    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3570    /// truncates to the single slot rotation picked, so a solo task whose one
3571    /// implementer hits quota mid-run used to have nothing else to try. This
3572    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3573    /// unrotated roster — which is the only place the *other* candidates in
3574    /// the machine's roster still exist once `implementers` has been cut down
3575    /// to size.
3576    ///
3577    /// Walks forward from just past the seat's own original position in the
3578    /// roster, never wrapping back to the front: a later candidate slot (say
3579    /// `beta`, the roster's second entry) must fall through to the *next*
3580    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3581    /// almost certainly a different candidate's own agent already — and once
3582    /// the roster's tail is exhausted there is nothing left to fall through
3583    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3584    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3585    /// let this retry that id forever. The loop keeps falling through until
3586    /// an attempt lands something other than `Quota` or the roster's tail
3587    /// runs out of untried ids, at which point the seat is left exactly as
3588    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3589    /// `QuotaLoss` recorded, the candidate failed/empty.
3590    ///
3591    /// `sent` is taken mutably and updated with the fallback agent's spec:
3592    /// `resume_unconfirmed_commands`, which runs after this and also reads
3593    /// `sent`, must see whichever agent actually ended up answering the seat
3594    /// — reading the stale, original spec there would check session
3595    /// eligibility against the wrong CLI and could hand a fallback agent's
3596    /// session id to the agent that just lost the seat to quota.
3597    ///
3598    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3599    /// — `self.seat` only reuses state when the agent id is unchanged, so
3600    /// handing it a different id already gets this for free. Reusing the old
3601    /// seat would resume a different CLI's session as if it were a
3602    /// continuation of this one.
3603    ///
3604    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3605    /// a quota loss cuts an agent off mid-turn, so anything already in the
3606    /// tree is unfinished work, not a completed candidate a re-ask would pay
3607    /// for twice. A dirty tree is rescued into a commit first (the same
3608    /// neutral-identity rescue `implement`'s own outcome loop gives every
3609    /// candidate) so the next agent starts clean.
3610    ///
3611    /// The new agent gets the implementer's full prompt and full
3612    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3613    /// it has no session and no context, and is implementing the task from
3614    /// nothing, unlike a resumed drop which is only restating work already
3615    /// done.
3616    ///
3617    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3618    /// `implement` event, never into `self.state.quota` — that is what
3619    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3620    /// go unspent, and a seat that ultimately recovered on its second or
3621    /// third agent is not the stalled panel that check exists to catch. Only
3622    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3623    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3624    /// outcome loop already has — this helper never pushes to it itself.
3625    async fn resume_seat_handovers(
3626        &mut self,
3627        results: &mut [(usize, SeatState, AgentOutcome)],
3628        sent: &mut [SeatJob],
3629        prompts: &Prompts,
3630        run_id: &str,
3631    ) -> Result<()> {
3632        let instruction = seeded_instruction(&self.state);
3633        let language = self.state.config.graph.language.clone();
3634        let github_language = self.state.config.graph.github_language.clone();
3635        let brief = self
3636            .state
3637            .advice
3638            .as_ref()
3639            .and_then(|a| a.synthesis.as_deref())
3640            .map(str::to_owned);
3641        let attachments = self.state.attachments.clone();
3642        for (wi, seat, out) in results.iter_mut() {
3643            // Who holds the other candidate seats of this wave right now
3644            // (earlier handovers already written back to `sent`).
3645            let others: BTreeSet<String> = sent
3646                .iter()
3647                .enumerate()
3648                .filter(|(j, _)| j != wi)
3649                .map(|(_, j)| j.spec.id.clone())
3650                .collect();
3651            let Some(job) = sent.get_mut(*wi) else {
3652                continue;
3653            };
3654            // Where the seat's own original agent sits in the roster — the
3655            // fallback walk starts just past here, never at the front, so a
3656            // later candidate slot's quota loss does not fall back onto an
3657            // earlier slot's own agent.
3658            let start = self
3659                .roles
3660                .implementer_roster
3661                .iter()
3662                .position(|s| s.id == job.spec.id)
3663                .unwrap_or(0);
3664            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3665            let mut fallback_attempt = 0usize;
3666            let mut prev: Option<FailClass> = None;
3667            while let Some(cur) = FailClass::of(&*out) {
3668                if !should_hand_over(prev.as_ref(), &cur) {
3669                    break;
3670                }
3671                let Some(next) =
3672                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3673                        .cloned()
3674                else {
3675                    break;
3676                };
3677                tried.insert(next.id.clone());
3678                fallback_attempt += 1;
3679
3680                let coordination = self.coordinate(&job.cwd).await?;
3681                if let Ok(r) = git::rescue_commit(
3682                    &job.cwd,
3683                    &format!(
3684                        "magi: candidate {} (uncommitted work before {} fallback)",
3685                        seat.key,
3686                        if cur == FailClass::Quota {
3687                            "quota"
3688                        } else {
3689                            "handover"
3690                        }
3691                    ),
3692                )
3693                .await
3694                {
3695                    self.state.note_withheld("implement", &r.withheld);
3696                }
3697                drop(coordination);
3698
3699                record_handover(
3700                    &mut self.state,
3701                    "implement",
3702                    &seat.key,
3703                    &seat.agent,
3704                    &next.id,
3705                    &cur,
3706                    &fail_reason(&*out),
3707                );
3708                prev = Some(cur.clone());
3709
3710                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3711                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3712                // Kept in sync on `sent` itself, not just the local retry: a
3713                // later helper (`resume_unconfirmed_commands`) reads `sent`
3714                // after this one returns and must see whichever agent is now
3715                // occupying the seat, not the one that just quota'd out —
3716                // otherwise it would judge session/continuation eligibility
3717                // by the wrong CLI and could resend a fallback's session id
3718                // to the agent that lost it the seat in the first place.
3719                job.spec = next.clone();
3720                let mut retry = job.clone();
3721                retry.seat = new_seat;
3722                retry.prompt = prompt::implement_in(
3723                    &instruction,
3724                    &job.cwd.to_string_lossy(),
3725                    &language,
3726                    &github_language,
3727                    brief.as_deref(),
3728                    &attachments,
3729                );
3730                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3731                let cache = self.state.config.cache_dir();
3732                let ctx = WaveCtx {
3733                    carry_seats: false,
3734                    pauses: Some((&self.pause, &self.interrupt)),
3735                    run: run_id,
3736                    node: "implement",
3737                    prompts,
3738                    cache: cache.as_deref(),
3739                    round: None,
3740                };
3741                let (fallback_seat, fallback_out) = run_one(
3742                    retry,
3743                    Arc::clone(&self.sem),
3744                    &ctx,
3745                    &mut self.state,
3746                    fallback_attempt,
3747                )
3748                .await;
3749                *seat = fallback_seat;
3750                *out = fallback_out;
3751            }
3752        }
3753        Ok(())
3754    }
3755
3756    /// Ask an implement seat's own CLI to confirm what it started, once, when
3757    /// its reply reported a command whose completion status it never
3758    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3759    /// that does and does not mean.
3760    ///
3761    /// The completion contract this task asks for, extended to `implement`
3762    /// with the same signal `continue_fix_report` reads for the fixer,
3763    /// rather than a keyword search over the reply or a hard requirement on
3764    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3765    /// a candidate's CLI turn ended cleanly while a test run it had started
3766    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3767    /// named in it at all is untouched by this: `commands` is empty, so
3768    /// there is nothing to be unconfirmed.
3769    ///
3770    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3771    /// this is not about recovering edits that might already be on disk, it
3772    /// is about a result the seat itself never vouched for, which resuming
3773    /// asks for regardless of what the tree already holds. Bounded to one
3774    /// attempt for the same reason `resume_undelivered` is — this is the
3775    /// most expensive node in the graph — and a seat that still cannot
3776    /// confirm on that attempt is left as whatever its (possibly still
3777    /// unconfirmed) reply says; this does not invent a new "failed" reason
3778    /// for a candidate that otherwise produced a real, committed change.
3779    async fn resume_unconfirmed_commands(
3780        &mut self,
3781        results: &mut [(usize, SeatState, AgentOutcome)],
3782        sent: &[SeatJob],
3783        prompts: &Prompts,
3784        run_id: &str,
3785    ) {
3786        for (wi, seat, out) in results.iter_mut() {
3787            let AgentOutcome::Ok(o) = &*out else {
3788                continue;
3789            };
3790            if !has_unconfirmed_command(&o.commands) {
3791                continue;
3792            }
3793            let Some(job) = sent.get(*wi) else { continue };
3794            if !has_context(&job.spec, seat, job.sessions) {
3795                self.state.event(
3796                    "implement",
3797                    format!(
3798                        "{}: the reply named a command whose own CLI never confirmed the exit \
3799                         status of, but there is no session left to resume",
3800                        seat.key
3801                    ),
3802                );
3803                continue;
3804            }
3805            self.state.event(
3806                "implement",
3807                format!(
3808                    "{}: the reply named a command whose own CLI never confirmed the exit \
3809                     status of; resuming the conversation",
3810                    seat.key
3811                ),
3812            );
3813            let mut retry = job.clone();
3814            retry.seat = seat.clone();
3815            retry.prompt = prompt::resume_incomplete(
3816                "a command in your last reply had no confirmed exit status",
3817            );
3818            retry.timeout = retry_budget(job.timeout, true);
3819            retry.stem = format!("{}-confirm", job.stem);
3820            let cache = self.state.config.cache_dir();
3821            let ctx = WaveCtx {
3822                carry_seats: false,
3823                pauses: Some((&self.pause, &self.interrupt)),
3824                run: run_id,
3825                node: "implement",
3826                prompts,
3827                cache: cache.as_deref(),
3828                round: None,
3829            };
3830            let (resumed_seat, resumed) =
3831                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3832            *seat = resumed_seat;
3833            *out = resumed;
3834        }
3835    }
3836
3837    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3838    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3839    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3840    /// that motivated this.
3841    ///
3842    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3843    /// own nudge loop already covers for judge/review/vote seats, and not a
3844    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3845    /// implement seats: here the CLI turn genuinely finished while the node's
3846    /// own work — the fixer's account of what it did — had not. Gated purely
3847    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3848    /// reply, never on any wording in it, so a fixer whose valid, first-try
3849    /// `FixReport` happens to mention having waited on a background test is
3850    /// never resumed — the `Ok(report)` branch at the call site returns
3851    /// before this is ever invoked.
3852    ///
3853    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3854    /// attempt ([`retry_budget`]), nothing attempted once the session is
3855    /// gone, and a quota hit ends the loop immediately rather than retrying a
3856    /// rate limit that fails the same way again.
3857    async fn continue_fix_report(
3858        &mut self,
3859        mut seat: SeatState,
3860        parse_err: String,
3861        job: &SeatJob,
3862        prompts: &Prompts,
3863        run_id: &str,
3864        round: usize,
3865    ) -> (
3866        SeatState,
3867        Option<FixReport>,
3868        Option<String>,
3869        ContinuationRecord,
3870    ) {
3871        let mut last_err = parse_err;
3872        let mut cumulative_wait_ms = 0u64;
3873        let mut attempts = 0usize;
3874        loop {
3875            if !has_context(&job.spec, &seat, job.sessions) {
3876                self.state.event(
3877                    "fix",
3878                    format!(
3879                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3880                         session left to resume into"
3881                    ),
3882                );
3883                let outcome = if attempts == 0 {
3884                    ContinuationOutcome::NoSession
3885                } else {
3886                    ContinuationOutcome::Exhausted
3887                };
3888                return (
3889                    seat,
3890                    None,
3891                    Some(format!("unparsable fix report: {last_err}")),
3892                    ContinuationRecord {
3893                        attempts,
3894                        cumulative_wait_ms,
3895                        outcome,
3896                    },
3897                );
3898            }
3899            if attempts >= MAX_FIX_CONTINUATIONS {
3900                self.state.event(
3901                    "fix",
3902                    format!(
3903                        "round {round}: fixer's reply still had no adoption report after \
3904                         {attempts} continuation(s) ({last_err}); giving up"
3905                    ),
3906                );
3907                return (
3908                    seat,
3909                    None,
3910                    Some(format!(
3911                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3912                    )),
3913                    ContinuationRecord {
3914                        attempts,
3915                        cumulative_wait_ms,
3916                        outcome: ContinuationOutcome::Exhausted,
3917                    },
3918                );
3919            }
3920            attempts += 1;
3921            self.state.event(
3922                "fix",
3923                format!(
3924                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3925                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3926                ),
3927            );
3928            let mut retry = job.clone();
3929            retry.seat = seat.clone();
3930            retry.prompt = prompt::resume_incomplete(&last_err);
3931            retry.timeout = retry_budget(job.timeout, true);
3932            retry.stem = format!("{}-continue{attempts}", job.stem);
3933            let cache = self.state.config.cache_dir();
3934            let ctx = WaveCtx {
3935                carry_seats: false,
3936                pauses: Some((&self.pause, &self.interrupt)),
3937                run: run_id,
3938                node: "fix",
3939                prompts,
3940                cache: cache.as_deref(),
3941                round: Some(round),
3942            };
3943            let (resumed_seat, resumed_out) = run_one(
3944                retry,
3945                Arc::clone(&self.sem),
3946                &ctx,
3947                &mut self.state,
3948                attempts,
3949            )
3950            .await;
3951            seat = resumed_seat;
3952            match resumed_out {
3953                AgentOutcome::Ok(o) => {
3954                    cumulative_wait_ms += o.duration_ms;
3955                    match verdict::extract_json::<FixReport>(&o.text) {
3956                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3957                            self.state.event(
3958                                "fix",
3959                                format!(
3960                                    "round {round}: fixer's adoption report recovered after \
3961                                     {attempts} continuation(s)"
3962                                ),
3963                            );
3964                            return (
3965                                seat,
3966                                Some(report),
3967                                None,
3968                                ContinuationRecord {
3969                                    attempts,
3970                                    cumulative_wait_ms,
3971                                    outcome: ContinuationOutcome::Resumed,
3972                                },
3973                            );
3974                        }
3975                        // The report parsed, but this same reply's own
3976                        // CommandEvidence — the identical record `state.jobs`
3977                        // renders — names a command whose CLI never
3978                        // confirmed an exit status. Read together, that is
3979                        // not a resolved answer: keep nudging rather than
3980                        // accept a report standing next to a command the
3981                        // seat's own CLI cannot vouch for.
3982                        Ok(_) => {
3983                            last_err = "the reply parsed, but it reported a command whose own CLI \
3984                                 never confirmed an exit status"
3985                                .to_owned();
3986                        }
3987                        Err(e) => last_err = e.to_string(),
3988                    }
3989                }
3990                AgentOutcome::Quota(o) => {
3991                    cumulative_wait_ms += o.duration_ms;
3992                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3993                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3994                    self.state.quota.push(loss);
3995                    self.state.event(
3996                        "fix",
3997                        format!(
3998                            "round {round}: continuation rate limited (quota); not retrying now"
3999                        ),
4000                    );
4001                    return (
4002                        seat,
4003                        None,
4004                        Some("rate limited (quota) while recovering the fix report".to_owned()),
4005                        ContinuationRecord {
4006                            attempts,
4007                            cumulative_wait_ms,
4008                            outcome: ContinuationOutcome::QuotaLost,
4009                        },
4010                    );
4011                }
4012                AgentOutcome::Dropped(o) => {
4013                    cumulative_wait_ms += o.duration_ms;
4014                    let why = o
4015                        .dropped
4016                        .as_ref()
4017                        .map(|d| d.why.as_str())
4018                        .unwrap_or("the CLI ended the stream without delivering its answer");
4019                    last_err = format!("the CLI dropped the stream ({why})");
4020                }
4021                AgentOutcome::Failed(e) => last_err = e,
4022                // The continuation was never started; the report stays lost
4023                // and the caller records that, without any handover.
4024                AgentOutcome::ResourceBlocked(e) => {
4025                    return (
4026                        seat,
4027                        None,
4028                        Some(format!("the continuation was not started: {e}")),
4029                        ContinuationRecord {
4030                            attempts: attempts.saturating_sub(1),
4031                            cumulative_wait_ms,
4032                            outcome: ContinuationOutcome::NoSession,
4033                        },
4034                    );
4035                }
4036            }
4037        }
4038    }
4039
4040    fn after_implement(&mut self) -> Result<()> {
4041        if !self.state.candidate_preservation.is_empty() {
4042            bail!("candidate preservation incomplete; judging is unsafe");
4043        }
4044        // Scan every candidate patch once the set is complete.
4045        if self.state.leaks.is_empty() {
4046            let cfg = self.state.config.blind.clone();
4047            let mut leaks = Vec::new();
4048            for c in &self.state.candidates {
4049                let Some(patch) =
4050                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4051                else {
4052                    continue;
4053                };
4054                leaks.extend(blind::scan(
4055                    &format!("candidate {} patch", c.label),
4056                    &patch,
4057                    &cfg.vendor_tokens,
4058                ));
4059            }
4060            if !leaks.is_empty() {
4061                let summary = leaks
4062                    .iter()
4063                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
4064                    .collect::<Vec<_>>()
4065                    .join(", ");
4066                match cfg.on_leak {
4067                    LeakPolicy::Fail => {
4068                        self.state.status = RunStatus::Failed;
4069                        self.state
4070                            .event("blind", format!("vendor text in a patch: {summary}"));
4071                        self.state.leaks = leaks;
4072                        self.state.save()?;
4073                        self.settle_questions();
4074                        bail!(
4075                            "blind.on_leak = \"fail\" and vendor text reached a \
4076                             judged patch: {summary}"
4077                        );
4078                    }
4079                    LeakPolicy::Redact => self.state.event(
4080                        "blind",
4081                        format!("redacting vendor text for judging: {summary}"),
4082                    ),
4083                    LeakPolicy::Warn => self.state.event(
4084                        "blind",
4085                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
4086                    ),
4087                }
4088                self.state.leaks = leaks;
4089            }
4090        }
4091
4092        if self.state.viable().is_empty() {
4093            if self.state.all_candidates_verified_noop() {
4094                // Every candidate agreed, with evidence the adoption guard
4095                // accepted, that nothing belongs in this worktree. That is
4096                // not the same fact as a candidate that simply failed to
4097                // write anything, and settling it as an ordinary `Failed`
4098                // (see `SCHEMA`'s doc for schema 10) is what let two of
4099                // task 391f's attempts burn a retry each re-discovering the
4100                // same already-landed fix. Terminal either way, so `judge`
4101                // must never run over an empty candidate set — unlike the
4102                // `Failed` branch below this returns `Ok`, not an error:
4103                // nothing here failed.
4104                self.state.status = RunStatus::VerifiedNoop;
4105                self.state.save()?;
4106                self.settle_questions();
4107                return Ok(());
4108            }
4109            self.state.status = RunStatus::Failed;
4110            self.state.save()?;
4111            self.settle_questions();
4112            bail!("no candidate produced a change; nothing to judge");
4113        }
4114        self.state.status = RunStatus::Judging;
4115        self.state.save()?;
4116        Ok(())
4117    }
4118
4119    // --------------------------------------------------------------- judge
4120
4121    async fn judge(&mut self) -> Result<()> {
4122        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4123        // agent files with `magi task add` name the run that paid for it. The
4124        // prompt overlay is cloned alongside it because the waves borrow it
4125        // while `self` is mutably borrowed by the node's own bookkeeping.
4126        let run_id = self.state.id.clone();
4127        let prompts = self.state.config.prompts.clone();
4128        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4129            return Ok(());
4130        }
4131        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4132        if viable.len() == 1 {
4133            // Recorded so this is a one-time event: `judgements` stays empty
4134            // either way, which without this flag is indistinguishable from
4135            // "not yet judged" on the next reentry — and status is left
4136            // untouched, so a later node's conclusion (e.g. `Blocked` after
4137            // the review budget ran out) survives a resume instead of being
4138            // clobbered back to `Judging` by this node running again.
4139            self.state.judge_skipped = true;
4140            self.state.event(
4141                "judge",
4142                format!(
4143                    "only candidate {} produced a change; judging skipped",
4144                    viable[0].label
4145                ),
4146            );
4147            self.state.save()?;
4148            return Ok(());
4149        }
4150        self.state.status = RunStatus::Judging;
4151
4152        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4153        let language = self.state.config.graph.language.clone();
4154        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4155        let sessions = self.state.config.graph.sessions;
4156        let artifacts = agent::artifacts_dir(&self.state.dir());
4157        let root = self.state.worktree_root();
4158        let base_short = short(&self.state.base_commit);
4159
4160        let mut jobs = Vec::new();
4161        let mut orders = Vec::new();
4162        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4163            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4164            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4165            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4166            let seat_key = format!("judge-{}", j + 1);
4167            let seat = self.seat(&seat_key, &spec.id);
4168            jobs.push(SeatJob {
4169                prompt: prompt::judge(
4170                    &self.state.instruction,
4171                    &views,
4172                    self.roles.judges.len(),
4173                    &base_short,
4174                    &language,
4175                ),
4176                spec,
4177                seat,
4178                cwd: root.join(format!("judge-{}", j + 1)),
4179                timeout,
4180                allow_write: false,
4181                sessions,
4182                artifacts: artifacts.clone(),
4183                stem: format!("judge-{}", j + 1),
4184                handover: None,
4185            });
4186        }
4187
4188        self.state.event(
4189            "judge",
4190            format!(
4191                "{} judges ranking {} candidates blind",
4192                jobs.len(),
4193                viable.len()
4194            ),
4195        );
4196        let labels_for_check = labels.clone();
4197        let mut quota_losses = Vec::new();
4198        let cache = self.state.config.cache_dir();
4199        let ctx = WaveCtx {
4200            carry_seats: false,
4201            pauses: Some((&self.pause, &self.interrupt)),
4202            run: &run_id,
4203            node: "judge",
4204            prompts: &prompts,
4205            cache: cache.as_deref(),
4206            round: None,
4207        };
4208        let results = ask_json_wave::<Ranking>(
4209            jobs,
4210            Arc::clone(&self.sem),
4211            self.state.config.graph.retries,
4212            &self.roles.judge_roster,
4213            &ctx,
4214            &mut quota_losses,
4215            &mut self.state,
4216            &move |r: &Ranking| r.validate(&labels_for_check),
4217        )
4218        .await;
4219        if self.state.resource_block.is_some() {
4220            return self.stop_for_resource();
4221        }
4222        self.state.quota.extend(quota_losses);
4223
4224        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4225            let agent_id = seat.agent.clone();
4226            self.state.seats.insert(seat.key.clone(), seat);
4227            let mut record = Judgement {
4228                judge: j + 1,
4229                seat: format!("judge-{}", j + 1),
4230                agent: agent_id,
4231                ranking: Vec::new(),
4232                reasons: BTreeMap::new(),
4233                confidence: None,
4234                strengths: BTreeMap::new(),
4235                order: orders[j].clone(),
4236                failed: None,
4237                duration_ms: 0,
4238            };
4239            match res {
4240                Ok((ranking, out)) => {
4241                    record.ranking = ranking.normalized();
4242                    record.reasons = ranking.reasons;
4243                    record.strengths = ranking.strengths;
4244                    record.confidence = ranking.confidence;
4245                    record.duration_ms = out.duration_ms;
4246                    self.state.event(
4247                        "judge",
4248                        format!(
4249                            "judge {} ranked {}",
4250                            j + 1,
4251                            record.ranking.iter().collect::<String>()
4252                        ),
4253                    );
4254                }
4255                Err(e) => {
4256                    record.failed = Some(e.to_string());
4257                    self.state
4258                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4259                }
4260            }
4261            self.state.judgements.push(record);
4262            self.state.save()?;
4263        }
4264        Ok(())
4265    }
4266
4267    // ---------------------------------------------------------- deliberate
4268
4269    async fn deliberate(&mut self) -> Result<()> {
4270        if self.hold_base_sync().await? {
4271            return Ok(());
4272        }
4273        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4274        // agent files with `magi task add` name the run that paid for it. The
4275        // prompt overlay is cloned alongside it because the waves borrow it
4276        // while `self` is mutably borrowed by the node's own bookkeeping.
4277        let run_id = self.state.id.clone();
4278        let prompts = self.state.config.prompts.clone();
4279        let rounds = self.state.config.graph.deliberate_rounds;
4280        // Resume unfinished rounds after an outage, but never reopen a tally.
4281        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4282            return Ok(());
4283        }
4284        let tops: Vec<char> = self
4285            .state
4286            .judgements
4287            .iter()
4288            .filter_map(|j| j.ranking.first().copied())
4289            .collect();
4290        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4291            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4292                self.state.event(
4293                    "deliberate",
4294                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4295                );
4296            }
4297            self.state.status = RunStatus::Voting;
4298            self.state.save()?;
4299            return Ok(());
4300        }
4301
4302        self.state.status = RunStatus::Deliberating;
4303        self.state.event(
4304            "deliberate",
4305            format!(
4306                "split: first choices were {} — opening {rounds} round(s)",
4307                tops.iter().collect::<String>()
4308            ),
4309        );
4310
4311        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4312        let language = self.state.config.graph.language.clone();
4313        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4314        let sessions = self.state.config.graph.sessions;
4315        let artifacts = agent::artifacts_dir(&self.state.dir());
4316        let root = self.state.worktree_root();
4317        let base_short = short(&self.state.base_commit);
4318
4319        // Judges argue in sequence so that a turn can answer the one before it;
4320        // that is the difference between deliberation and three parallel
4321        // monologues.
4322        for round in self.state.deliberation.len() + 1..=rounds {
4323            let mut turns: Vec<DeliberationTurn> = Vec::new();
4324            let mut round_losses = Vec::new();
4325            let mut asked = 0;
4326            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4327                if self.state.judgements[j].failed.is_some() {
4328                    continue;
4329                }
4330                let seat_key = format!("judge-{}", j + 1);
4331                let spec = self.occupant(&seat_key, spec);
4332                let mut seat = self.seat(&seat_key, &spec.id);
4333                let transcript = self.transcript(&turns, j);
4334                let build = |context: Option<&str>| {
4335                    prompt::deliberate(
4336                        &self.state.instruction,
4337                        context,
4338                        &transcript,
4339                        round,
4340                        rounds,
4341                        &language,
4342                    )
4343                };
4344                let block = self.candidate_block(&viable, &base_short);
4345                let full = build(Some(&block));
4346                let text = if has_context(&spec, &seat, sessions) {
4347                    build(None)
4348                } else {
4349                    full.clone()
4350                };
4351                let job = SeatJob {
4352                    spec,
4353                    seat: seat.clone(),
4354                    prompt: text,
4355                    cwd: root.join(format!("judge-{}", j + 1)),
4356                    timeout,
4357                    allow_write: false,
4358                    sessions,
4359                    artifacts: artifacts.clone(),
4360                    stem: format!("delib-{round}-judge-{}", j + 1),
4361                    handover: Some(full),
4362                };
4363                let cache = self.state.config.cache_dir();
4364                let ctx = WaveCtx {
4365                    carry_seats: false,
4366                    pauses: Some((&self.pause, &self.interrupt)),
4367                    run: &run_id,
4368                    node: "deliberate",
4369                    prompts: &prompts,
4370                    cache: cache.as_deref(),
4371                    round: None,
4372                };
4373                // A turn is never nudged (`retries` 0): a failed seat is
4374                // handed to the next roster agent, which gets the full
4375                // context. An empty answer is a turn, not a failure.
4376                let mut losses = Vec::new();
4377                asked += 1;
4378                let mut results = ask_wave_with::<String>(
4379                    vec![job],
4380                    Arc::clone(&self.sem),
4381                    0,
4382                    &self.roles.judge_roster,
4383                    &ctx,
4384                    &mut losses,
4385                    &mut self.state,
4386                    &|text: &str| {
4387                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4388                    },
4389                )
4390                .await;
4391                if self.state.resource_block.is_some() {
4392                    return self.stop_for_resource();
4393                }
4394                round_losses.extend(losses.iter().cloned());
4395                self.state.quota.extend(losses);
4396                let (updated, res, _) = results.pop().expect("one job in, one result out");
4397                seat = updated;
4398                let agent_id = seat.agent.clone();
4399                self.state.seats.insert(seat.key.clone(), seat);
4400                let body = match res {
4401                    Ok((body, _)) => body,
4402                    // Skip the seat; a CLI's raw error JSON is never read as
4403                    // this judge's position.
4404                    Err(e) => {
4405                        self.state
4406                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4407                        continue;
4408                    }
4409                };
4410                let tentative = verdict::extract_json::<Position>(&body)
4411                    .ok()
4412                    .and_then(|p| p.tentative)
4413                    .and_then(|s| s.trim().chars().next())
4414                    .map(|c| c.to_ascii_uppercase());
4415                self.state.event(
4416                    "deliberate",
4417                    format!(
4418                        "round {round}: judge {} now favours {}",
4419                        j + 1,
4420                        tentative.map_or("—".to_owned(), |c| c.to_string())
4421                    ),
4422                );
4423                turns.push(DeliberationTurn {
4424                    judge: j + 1,
4425                    agent: agent_id,
4426                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4427                    tentative,
4428                });
4429            }
4430            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4431                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4432                return self.stop_for_resource();
4433            }
4434            self.state
4435                .deliberation
4436                .push(DeliberationRound { round, turns });
4437            self.state.save()?;
4438        }
4439
4440        self.state.status = RunStatus::Voting;
4441        self.state.save()?;
4442        Ok(())
4443    }
4444
4445    // ---------------------------------------------------------------- vote
4446
4447    async fn vote(&mut self) -> Result<()> {
4448        if self.hold_base_sync().await? {
4449            return Ok(());
4450        }
4451        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4452        // agent files with `magi task add` name the run that paid for it. The
4453        // prompt overlay is cloned alongside it because the waves borrow it
4454        // while `self` is mutably borrowed by the node's own bookkeeping.
4455        let run_id = self.state.id.clone();
4456        let prompts = self.state.config.prompts.clone();
4457        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4458            return Ok(());
4459        }
4460        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4461        if viable.len() == 1 {
4462            return Ok(());
4463        }
4464        self.state.status = RunStatus::Voting;
4465
4466        let language = self.state.config.graph.language.clone();
4467        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4468        let sessions = self.state.config.graph.sessions;
4469        let artifacts = agent::artifacts_dir(&self.state.dir());
4470        let root = self.state.worktree_root();
4471        let base_short = short(&self.state.base_commit);
4472        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4473
4474        let mut jobs = Vec::new();
4475        let mut seats_at = Vec::new();
4476        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4477            if self
4478                .state
4479                .judgements
4480                .get(j)
4481                .is_some_and(|r| r.failed.is_some())
4482            {
4483                continue;
4484            }
4485            let seat_key = format!("judge-{}", j + 1);
4486            let spec = self.occupant(&seat_key, spec);
4487            let seat = self.seat(&seat_key, &spec.id);
4488            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4489            let text = if has_context(&spec, &seat, sessions) {
4490                prompt::final_vote(&viable, &language)
4491            } else {
4492                full.clone()
4493            };
4494            jobs.push(SeatJob {
4495                spec,
4496                seat,
4497                prompt: text,
4498                cwd: root.join(format!("judge-{}", j + 1)),
4499                timeout,
4500                allow_write: false,
4501                sessions,
4502                artifacts: artifacts.clone(),
4503                stem: format!("vote-judge-{}", j + 1),
4504                handover: Some(full),
4505            });
4506            seats_at.push(j);
4507        }
4508
4509        self.state.event(
4510            "vote",
4511            format!(
4512                "collecting {} final votes one by one, privately",
4513                jobs.len()
4514            ),
4515        );
4516        let allowed = viable.clone();
4517        let mut quota_losses = Vec::new();
4518        let cache = self.state.config.cache_dir();
4519        let ctx = WaveCtx {
4520            carry_seats: false,
4521            pauses: Some((&self.pause, &self.interrupt)),
4522            run: &run_id,
4523            node: "vote",
4524            prompts: &prompts,
4525            cache: cache.as_deref(),
4526            round: None,
4527        };
4528        let results = ask_json_wave::<FinalVote>(
4529            jobs,
4530            Arc::clone(&self.sem),
4531            self.state.config.graph.retries,
4532            &self.roles.judge_roster,
4533            &ctx,
4534            &mut quota_losses,
4535            &mut self.state,
4536            &move |v: &FinalVote| match v.label() {
4537                Some(c) if allowed.contains(&c) => Ok(()),
4538                other => bail!("vote {other:?} is not one of {allowed:?}"),
4539            },
4540        )
4541        .await;
4542        if self.state.resource_block.is_some() {
4543            return self.stop_for_resource();
4544        }
4545        self.state.quota.extend(quota_losses);
4546
4547        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4548            let agent_id = seat.agent.clone();
4549            self.state.seats.insert(seat.key.clone(), seat);
4550            let initial = self
4551                .state
4552                .judgements
4553                .get(j)
4554                .and_then(|r| r.ranking.first().copied());
4555            let mut record = VoteRecord {
4556                judge: j + 1,
4557                agent: agent_id,
4558                vote: None,
4559                reason: String::new(),
4560                changed: false,
4561            };
4562            match res {
4563                Ok((v, _)) => {
4564                    record.vote = v.label();
4565                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4566                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4567                    self.state.event(
4568                        "vote",
4569                        format!(
4570                            "judge {} voted {}{}",
4571                            j + 1,
4572                            record.vote.unwrap_or('?'),
4573                            if record.changed { " (changed)" } else { "" }
4574                        ),
4575                    );
4576                }
4577                Err(e) => {
4578                    self.state
4579                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4580                }
4581            }
4582            self.state.votes.push(record);
4583            self.state.save()?;
4584        }
4585        Ok(())
4586    }
4587
4588    // --------------------------------------------------------------- tally
4589
4590    fn tally(&mut self) -> Result<()> {
4591        if self.state.tally.is_some() {
4592            return Ok(());
4593        }
4594        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4595        let tops: Vec<char> = self
4596            .state
4597            .judgements
4598            .iter()
4599            .filter_map(|j| j.ranking.first().copied())
4600            .collect();
4601        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4602
4603        // A judge whose private vote failed still counted once, in the initial
4604        // ranking; using it beats discarding a whole seat.
4605        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4606        let mut cast: Vec<char> = Vec::new();
4607        for (i, j) in self.state.judgements.iter().enumerate() {
4608            let vote = self
4609                .state
4610                .votes
4611                .iter()
4612                .find(|v| v.judge == i + 1)
4613                .and_then(|v| v.vote)
4614                .or_else(|| j.ranking.first().copied());
4615            if let Some(v) = vote {
4616                *first_choice.entry(v).or_insert(0) += 1;
4617                cast.push(v);
4618            }
4619        }
4620
4621        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4622        for j in &self.state.judgements {
4623            let n = j.ranking.len();
4624            for (pos, label) in j.ranking.iter().enumerate() {
4625                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4626            }
4627        }
4628
4629        let best = first_choice.values().copied().max().unwrap_or(0);
4630        let mut leaders: Vec<char> = first_choice
4631            .iter()
4632            .filter(|(_, v)| **v == best)
4633            .map(|(k, _)| *k)
4634            .collect();
4635        let mut tie_break = None;
4636        if leaders.len() > 1 {
4637            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4638            let borda_leaders: Vec<char> = leaders
4639                .iter()
4640                .copied()
4641                .filter(|l| borda[l] == top_borda)
4642                .collect();
4643            tie_break = Some(if borda_leaders.len() == 1 {
4644                format!(
4645                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4646                    leaders.len()
4647                )
4648            } else {
4649                format!(
4650                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4651                    leaders.len()
4652                )
4653            });
4654            leaders = borda_leaders;
4655            leaders.sort_unstable();
4656        }
4657        let winner = *leaders
4658            .first()
4659            .or(viable.first())
4660            .context("no candidate to declare a winner from")?;
4661
4662        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4663        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4664        let deliberated = !self.state.deliberation.is_empty();
4665
4666        // Whose verdict is this? A rate-limited seat is absent even if it
4667        // ranked before the limit hit, so presence is measured against the
4668        // recorded losses, not just "did a ranking ever appear".
4669        let quota_seats: std::collections::BTreeSet<&str> =
4670            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4671        let mut present = 0usize;
4672        for (i, j) in self.state.judgements.iter().enumerate() {
4673            if quota_seats.contains(j.seat.as_str()) {
4674                continue;
4675            }
4676            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4677            let voted = self
4678                .state
4679                .votes
4680                .iter()
4681                .any(|v| v.judge == i + 1 && v.vote.is_some());
4682            if ranked || voted {
4683                present += 1;
4684            }
4685        }
4686        // Strict majority of the configured panel. A bare majority is real
4687        // signal we can act on, while a minority verdict must never stand in
4688        // for a healthy one. A one-candidate run needs no panel at all, and
4689        // `judges` stays `0` rather than the roster size a panel that never
4690        // sat would otherwise be credited with.
4691        let needs_quorum = viable.len() > 1;
4692        let judges_total = if needs_quorum {
4693            self.roles.judges.len()
4694        } else {
4695            0
4696        };
4697        let quorum = if needs_quorum {
4698            judges_total / 2 + 1
4699        } else {
4700            0
4701        };
4702        let met_quorum = !needs_quorum || present >= quorum;
4703        let uncontested = (!needs_quorum).then(|| {
4704            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4705        });
4706
4707        self.state.event(
4708            "tally",
4709            match &uncontested {
4710                Some(reason) => format!("winner {winner} — {reason}"),
4711                None => format!(
4712                    "winner {winner} — votes {} | initial {} | {} changed | \
4713                     {present}/{judges_total} judges{}",
4714                    first_choice
4715                        .iter()
4716                        .map(|(k, v)| format!("{k}:{v}"))
4717                        .collect::<Vec<_>>()
4718                        .join(" "),
4719                    if unanimous_initial {
4720                        "unanimous"
4721                    } else {
4722                        "split"
4723                    },
4724                    changed_votes,
4725                    if met_quorum {
4726                        String::new()
4727                    } else {
4728                        format!(" — below quorum ({quorum} required)")
4729                    },
4730                ),
4731            },
4732        );
4733        if !met_quorum {
4734            self.state.event(
4735                "stall",
4736                format!(
4737                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4738                     the run stops here, resumable"
4739                ),
4740            );
4741        }
4742        self.state.tally = Some(Tally {
4743            first_choice,
4744            borda,
4745            winner,
4746            rankings: tops.len(),
4747            unanimous_initial,
4748            deliberated,
4749            changed_votes,
4750            unanimous_final,
4751            tie_break,
4752            judges: judges_total,
4753            present,
4754            quorum,
4755            met_quorum,
4756            uncontested,
4757        });
4758        self.state.status = if met_quorum {
4759            RunStatus::Reviewing
4760        } else {
4761            RunStatus::Stalled
4762        };
4763        self.state.save()?;
4764        Ok(())
4765    }
4766
4767    // ------------------------------------------------------------- recover
4768
4769    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4770    /// resumed toward completion once the transient cause clears.
4771    ///
4772    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4773    /// it toward the quorum, which is exactly the set of seats whose absence
4774    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4775    /// not depend on which node happened to hit the limit), or an ordinary
4776    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4777    /// seat is never disturbed.
4778    ///
4779    /// A seat that now answers with a usable ranking is "recovered": its
4780    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4781    /// `tally` counts it present again), and its vote re-collected. A seat that
4782    /// still fails keeps its loss and stays absent.
4783    ///
4784    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4785    /// to review/gate/merge), `false` when it is still below quorum (the run
4786    /// stays `Stalled`, still resumable for a later retry).
4787    #[allow(clippy::too_many_lines)]
4788    async fn recover_stall(&mut self) -> Result<bool> {
4789        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4790        // agent files with `magi task add` name the run that paid for it. The
4791        // prompt overlay is cloned alongside it because the waves borrow it
4792        // while `self` is mutably borrowed by the node's own bookkeeping.
4793        let run_id = self.state.id.clone();
4794        let prompts = self.state.config.prompts.clone();
4795        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4796        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4797        // is what keeps a non-quota collapse — or a quota loss recorded at the
4798        // deliberate node — from being a permanent dead-end on `--resume`.
4799        let quota_seats: BTreeSet<&str> =
4800            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4801        let absent: Vec<String> = self
4802            .state
4803            .judgements
4804            .iter()
4805            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4806            .map(|j| j.seat.clone())
4807            .collect();
4808        if absent.is_empty() {
4809            return Ok(false);
4810        }
4811        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4812        if viable.len() <= 1 {
4813            return Ok(false);
4814        }
4815        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4816        let language = self.state.config.graph.language.clone();
4817        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4818        let sessions = self.state.config.graph.sessions;
4819        let artifacts = agent::artifacts_dir(&self.state.dir());
4820        let root = self.state.worktree_root();
4821        let base_short = short(&self.state.base_commit);
4822        let candidates: Vec<Candidate> = viable.clone();
4823
4824        // Map each absent seat key to its 0-based position in `roles.judges`.
4825        let mut positions: Vec<usize> = absent
4826            .iter()
4827            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4828            .collect();
4829        if positions.is_empty() {
4830            return Ok(false);
4831        }
4832        positions.sort_unstable();
4833        positions.dedup();
4834
4835        // Re-rank the lost seats, one blind prompt each.
4836        let mut judge_jobs = Vec::new();
4837        for &j in &positions {
4838            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4839            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4840            let seat_key = format!("judge-{}", j + 1);
4841            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4842            let seat = self.seat(&seat_key, &spec.id);
4843            judge_jobs.push(SeatJob {
4844                spec,
4845                seat,
4846                prompt: prompt::judge(
4847                    &self.state.instruction,
4848                    &views,
4849                    self.roles.judges.len(),
4850                    &base_short,
4851                    &language,
4852                ),
4853                cwd: root.join(seat_key),
4854                timeout,
4855                allow_write: false,
4856                sessions,
4857                artifacts: artifacts.clone(),
4858                stem: format!("judge-{}-recover", j + 1),
4859                handover: None,
4860            });
4861        }
4862
4863        let labels_for_check = labels.clone();
4864        let mut judge_losses = Vec::new();
4865        let retries = self.state.config.graph.retries;
4866        let cache = self.state.config.cache_dir();
4867        let ctx = WaveCtx {
4868            carry_seats: false,
4869            pauses: Some((&self.pause, &self.interrupt)),
4870            run: &run_id,
4871            node: "judge",
4872            prompts: &prompts,
4873            cache: cache.as_deref(),
4874            round: None,
4875        };
4876        let results = ask_json_wave::<Ranking>(
4877            judge_jobs,
4878            Arc::clone(&self.sem),
4879            retries,
4880            &self.roles.judge_roster,
4881            &ctx,
4882            &mut judge_losses,
4883            &mut self.state,
4884            &move |r: &Ranking| r.validate(&labels_for_check),
4885        )
4886        .await;
4887        if self.state.resource_block.is_some() {
4888            self.stop_for_resource()?;
4889            return Ok(false);
4890        }
4891
4892        // Refresh the judgement of every seat that ranked again.
4893        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4894        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4895            let agent_id = seat.agent.clone();
4896            self.state.seats.insert(seat.key.clone(), seat);
4897            let record = &mut self.state.judgements[j];
4898            match res {
4899                Ok((ranking, out)) => {
4900                    record.agent = agent_id;
4901                    record.ranking = ranking.normalized();
4902                    record.reasons = ranking.reasons;
4903                    record.strengths = ranking.strengths;
4904                    record.confidence = ranking.confidence;
4905                    record.failed = None;
4906                    record.duration_ms = out.duration_ms;
4907                    recovered.insert(j);
4908                    self.state.event(
4909                        "recover",
4910                        format!("judge {} ranked again after the limit", j + 1),
4911                    );
4912                }
4913                Err(e) => {
4914                    self.state
4915                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4916                }
4917            }
4918        }
4919
4920        // Re-ask the votes of the seats that recovered a ranking.
4921        let mut vote_jobs = Vec::new();
4922        let mut vote_pos: Vec<usize> = Vec::new();
4923        for &j in &recovered {
4924            let seat_key = format!("judge-{}", j + 1);
4925            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4926            let seat = self.seat(&seat_key, &spec.id);
4927            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4928            let text = if has_context(&spec, &seat, sessions) {
4929                prompt::final_vote(&labels, &language)
4930            } else {
4931                full.clone()
4932            };
4933            vote_jobs.push(SeatJob {
4934                spec,
4935                seat,
4936                prompt: text,
4937                cwd: root.join(seat_key),
4938                timeout,
4939                allow_write: false,
4940                sessions,
4941                artifacts: artifacts.clone(),
4942                stem: format!("vote-judge-{}-recover", j + 1),
4943                handover: Some(full),
4944            });
4945            vote_pos.push(j);
4946        }
4947        let allowed = labels.clone();
4948        let mut vote_losses = Vec::new();
4949        let vote_retries = self.state.config.graph.retries;
4950        let vote_cache = self.state.config.cache_dir();
4951        let ctx = WaveCtx {
4952            carry_seats: false,
4953            pauses: Some((&self.pause, &self.interrupt)),
4954            run: &run_id,
4955            node: "vote",
4956            prompts: &prompts,
4957            cache: vote_cache.as_deref(),
4958            round: None,
4959        };
4960        let votes = ask_json_wave::<FinalVote>(
4961            vote_jobs,
4962            Arc::clone(&self.sem),
4963            vote_retries,
4964            &self.roles.judge_roster,
4965            &ctx,
4966            &mut vote_losses,
4967            &mut self.state,
4968            &move |v: &FinalVote| match v.label() {
4969                Some(c) if allowed.contains(&c) => Ok(()),
4970                other => bail!("vote {other:?} is not one of {allowed:?}"),
4971            },
4972        )
4973        .await;
4974        if self.state.resource_block.is_some() {
4975            self.stop_for_resource()?;
4976            return Ok(false);
4977        }
4978        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4979            let agent_id = seat.agent.clone();
4980            self.state.seats.insert(seat.key.clone(), seat);
4981            match res {
4982                Ok((v, _)) => {
4983                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4984                        rec.vote = v.label();
4985                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4986                    } else {
4987                        self.state.votes.push(VoteRecord {
4988                            judge: j + 1,
4989                            agent: agent_id,
4990                            vote: v.label(),
4991                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4992                            changed: false,
4993                        });
4994                    }
4995                    self.state.event(
4996                        "recover",
4997                        format!("judge {} voted again after the limit", j + 1),
4998                    );
4999                }
5000                Err(e) => {
5001                    self.state
5002                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
5003                }
5004            }
5005        }
5006
5007        // A seat that ranked again is present even if its re-vote failed —
5008        // `tally` falls back to the initial ranking's first choice — so clear
5009        // its quota loss. Seats that still fail keep theirs and stay absent.
5010        let recovered_keys: BTreeSet<String> = recovered
5011            .iter()
5012            .map(|&j| format!("judge-{}", j + 1))
5013            .collect();
5014        self.state
5015            .quota
5016            .retain(|q| !recovered_keys.contains(&q.seat));
5017        // A seat that hit the limit again is a fresh loss, not the old one:
5018        // replace the stale entry so the history stays one-per-seat and the
5019        // daemon can tell this attempt's loss from a previous session's.
5020        for loss in judge_losses.into_iter().chain(vote_losses) {
5021            if recovered_keys.contains(&loss.seat) {
5022                continue;
5023            }
5024            self.state.quota.retain(|q| q.seat != loss.seat);
5025            self.state.quota.push(loss);
5026        }
5027
5028        // Recompute the verdict from the refreshed panel.
5029        self.state.tally = None;
5030        self.tally()?;
5031        Ok(self
5032            .state
5033            .tally
5034            .as_ref()
5035            .map(|t| t.met_quorum)
5036            .unwrap_or(false))
5037    }
5038
5039    // ---------------------------------------------------------------- graft
5040
5041    /// Port the strengths the judges attributed to the losing candidates onto
5042    /// the winner, once, between the tally and the review loop.
5043    ///
5044    /// Best-effort by construction: it returns nothing and fails nothing. A
5045    /// skip, a fixer that failed or hit quota, or a round that changed nothing
5046    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
5047    /// the run carries on as if the phase were off. See [`graft`] for the
5048    /// decisions and `AGENTS.md` for the invariants.
5049    ///
5050    /// The one exception is a rollback that could not be confirmed: the
5051    /// rejected commit may still be the winner's tip, so the run is blocked
5052    /// (resumable; the record stays `Running`, which makes a resume retry the
5053    /// restore) instead of reviewing, gating and merging it.
5054    async fn graft(&mut self) -> Result<()> {
5055        let Err(e) = self.graft_inner().await else {
5056            return Ok(());
5057        };
5058        if e.is::<crate::repo_lock::ResourceBlocked>() {
5059            return Err(e);
5060        }
5061        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
5062            self.state.status = RunStatus::Blocked;
5063            self.state.event("graft", format!("blocked: {e:#}"));
5064            self.state.save()?;
5065            return Err(e);
5066        }
5067        // An error after the `Running` record was saved may have left the
5068        // fixer's work in the tree: the record must not stay unsettled
5069        // (a later resume would reset past the review fixes), so confirm the
5070        // restore now, or block.
5071        if let Some(rec) = self.state.graft.clone()
5072            && !rec.outcome.settled()
5073        {
5074            if let Some(winner) = self.state.winner().cloned() {
5075                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
5076                    if re.is::<crate::repo_lock::ResourceBlocked>() {
5077                        return Err(re);
5078                    }
5079                    self.state.status = RunStatus::Blocked;
5080                    self.state.event("graft", format!("blocked: {re:#}"));
5081                    self.state.save()?;
5082                    return Err(re);
5083                }
5084            }
5085            if let Some(g) = self.state.graft.as_mut() {
5086                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
5087            }
5088        }
5089        self.state.event("graft", format!("skipped: {e:#}"));
5090        if let Err(e) = self.state.save() {
5091            tracing::warn!("save after graft: {e:#}");
5092        }
5093        Ok(())
5094    }
5095
5096    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
5097    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
5098    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
5099        let _coordination = self.coordinate(wt).await?;
5100        crate::recovery::restore_repair(wt, pre)
5101            .await
5102            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
5103    }
5104
5105    /// A skip is a run event, recorded once however often the node is re-entered.
5106    fn graft_skipped(&mut self, why: &str) {
5107        let message = format!("skipped: {why}");
5108        if !self
5109            .state
5110            .events
5111            .iter()
5112            .any(|e| e.node == "graft" && e.message == message)
5113        {
5114            self.state.event("graft", message);
5115        }
5116    }
5117
5118    async fn graft_inner(&mut self) -> Result<()> {
5119        // A round that was started and never finished (crash, kill): the
5120        // fixer may have left anything in the tree, so put the winner back.
5121        if let Some(rec) = self.state.graft.clone()
5122            && !rec.outcome.settled()
5123        {
5124            // Reviews exist only if the graft ended without its record being
5125            // settled (a swallowed error): the tree has moved on since, and
5126            // a reset to `pre` would discard the review fixes.
5127            if self.state.reviews.is_empty()
5128                && let Some(winner) = self.state.winner().cloned()
5129            {
5130                self.graft_restore(&winner.worktree, &rec.pre).await?;
5131            }
5132            if rec.outcome == GraftOutcome::WaitingForRestore {
5133                self.state.graft = None;
5134                self.state.save()?;
5135            } else {
5136                if let Some(g) = self.state.graft.as_mut() {
5137                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5138                }
5139                self.state.event(
5140                    "graft",
5141                    "rolled back: the previous round was interrupted; the winner is untouched",
5142                );
5143                self.state.save()?;
5144                return Ok(());
5145            }
5146        }
5147        let plan = match graft::plan(&self.state) {
5148            Ok(plan) => plan,
5149            Err(skip) => {
5150                if skip != graft::Skip::AlreadyDone {
5151                    self.graft_skipped(skip.reason());
5152                    self.state.save()?;
5153                }
5154                return Ok(());
5155            }
5156        };
5157        let Some(winner) = self.state.winner().cloned() else {
5158            return Ok(());
5159        };
5160        let wt = winner.worktree.clone();
5161        if !git::is_clean(&wt).await? {
5162            self.graft_skipped("the winner's worktree has uncommitted changes");
5163            self.state.save()?;
5164            return Ok(());
5165        }
5166        let pre = git::rev_parse(&wt, "HEAD").await?;
5167        // Saved before the fixer is called: a stop from here on is repaired
5168        // by the branch above on the next entry.
5169        self.state.graft = Some(GraftRecord {
5170            pre: pre.clone(),
5171            post: None,
5172            lines: None,
5173            limit: None,
5174            agent: None,
5175            outcome: GraftOutcome::Running,
5176            requested: graft::requested(&plan),
5177            adoption: None,
5178            adoption_error: None,
5179        });
5180        self.state.event(
5181            "graft",
5182            format!(
5183                "porting strengths of candidate(s) {} onto {}",
5184                plan.strengths
5185                    .iter()
5186                    .map(|(l, _, _)| l.to_string())
5187                    .collect::<Vec<_>>()
5188                    .join(", "),
5189                plan.winner
5190            ),
5191        );
5192        self.state.save()?;
5193
5194        let fix_prompt = prompt::graft(
5195            &self.state.instruction,
5196            plan.winner,
5197            &plan.strengths,
5198            &self.state.config.graph.language,
5199        );
5200        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5201        let sessions = self.state.config.graph.sessions;
5202        let artifacts = agent::artifacts_dir(&self.state.dir());
5203        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5204        let (_, seat, out) = self
5205            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5206                SeatJob {
5207                    prompt: fix_prompt.clone(),
5208                    spec,
5209                    seat,
5210                    cwd: wt.clone(),
5211                    timeout,
5212                    allow_write: true,
5213                    sessions,
5214                    artifacts: artifacts.clone(),
5215                    stem: "graft".to_owned(),
5216                    handover: None,
5217                }
5218            })
5219            .await;
5220        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5221            // Preserve the need to restore earlier fallback edits without
5222            // confusing an unstarted seat with an interrupted invocation.
5223            self.state.graft.as_mut().expect("graft record").outcome =
5224                GraftOutcome::WaitingForRestore;
5225            return self.stop_for_resource();
5226        }
5227        let mut why: Option<String> = None;
5228        match &out {
5229            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5230                why = Some("the fixer produced nothing usable".to_owned());
5231            }
5232            AgentOutcome::Ok(_) => {}
5233            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5234            AgentOutcome::Quota(o) => {
5235                self.state.quota.push(QuotaLoss::from_report(
5236                    seat.key.clone(),
5237                    "graft",
5238                    o.quota.as_ref(),
5239                ));
5240                why = Some("rate limited (quota)".to_owned());
5241            }
5242            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5243                why = Some(format!(
5244                    "the fixer failed: {}",
5245                    e.lines()
5246                        .next()
5247                        .unwrap_or("")
5248                        .chars()
5249                        .take(160)
5250                        .collect::<String>()
5251                ));
5252            }
5253        }
5254        let agent_id = seat.agent.clone();
5255        self.state.seats.insert(seat.key.clone(), seat);
5256
5257        let mut post = None;
5258        let mut lines = None;
5259        let _coordination = self.coordinate(&wt).await?;
5260        if why.is_none() {
5261            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5262            if let Ok(r) = &rescued {
5263                self.state.note_withheld("graft", &r.withheld);
5264            }
5265            let head = git::rev_parse(&wt, "HEAD").await?;
5266            // What is measured must be what gets reviewed and gated: a failed
5267            // rescue, or anything still uncommitted (withheld lockfiles
5268            // aside, which are never part of the tree), rejects the graft.
5269            let dirty = match &rescued {
5270                Ok(r) => git::status_porcelain(&wt)
5271                    .await
5272                    .map(|st| {
5273                        st.lines().any(|l| {
5274                            !r.withheld
5275                                .iter()
5276                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5277                        })
5278                    })
5279                    .unwrap_or(true),
5280                Err(_) => true,
5281            };
5282            if dirty {
5283                why = Some("the fixer left changes that could not be committed".to_owned());
5284            } else if head == pre {
5285                why = Some("the fixer changed nothing".to_owned());
5286            } else {
5287                lines = git::changed_lines(&wt, &pre, &head).await;
5288                post = Some(head);
5289            }
5290        }
5291        let (adoption, adoption_error) = match &out {
5292            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5293            _ => (None, None),
5294        };
5295        let requested = graft::requested(&plan);
5296        let outcome = match why {
5297            None => {
5298                self.state.event(
5299                    "graft",
5300                    format!(
5301                        "applied: {}",
5302                        lines.map_or_else(
5303                            || "changed lines unavailable".to_owned(),
5304                            |n| format!("{n} changed lines")
5305                        )
5306                    ),
5307                );
5308                GraftOutcome::Applied
5309            }
5310            Some(why) => {
5311                // Not settled until the restore is confirmed: the record
5312                // stays `Running`, so a resume retries it.
5313                crate::recovery::restore_repair(&wt, &pre)
5314                    .await
5315                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5316                self.state.event(
5317                    "graft",
5318                    format!("rolled back: {why}; the winner is untouched"),
5319                );
5320                GraftOutcome::RolledBack(why)
5321            }
5322        };
5323        self.state.graft = Some(GraftRecord {
5324            pre,
5325            post,
5326            lines,
5327            limit: None,
5328            agent: Some(agent_id),
5329            outcome,
5330            requested,
5331            adoption,
5332            adoption_error,
5333        });
5334        self.state.save()?;
5335        Ok(())
5336    }
5337
5338    // ----------------------------------------------------------------- fold
5339
5340    async fn fold_losers(&mut self) -> Result<()> {
5341        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5342            return Ok(());
5343        };
5344        if !self.state.candidate_preservation.is_empty() {
5345            bail!("candidate preservation incomplete; refusing cleanup");
5346        }
5347        let repo = self.state.repo.clone();
5348        let _coordination = self.coordinate(&repo).await?;
5349        let home = crate::run::home();
5350        let root = self.state.worktree_root();
5351        let isolated = crate::build_target::isolated(&self.state.config);
5352        let mut folded = Vec::new();
5353        for i in 0..self.state.candidates.len() {
5354            let c = &self.state.candidates[i];
5355            if c.label == winner || c.folded {
5356                continue;
5357            }
5358            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5359            let target = fold_target(&home, &root, &wt, isolated);
5360            remove_preserved_worktree(&repo, &wt).await?;
5361            reclaim_fold_target(&home, target);
5362            if git::branch_exists(&repo, &branch).await?
5363                && !git::branch_delete(&repo, &branch).await?
5364            {
5365                bail!("candidate branch removal refused");
5366            }
5367            self.state.candidates[i].folded = true;
5368            self.state.save()?;
5369            folded.push(label.to_string());
5370        }
5371        // The judges are finished; their checkouts are pure cost from here.
5372        for j in 1..=self.roles.judges.len() {
5373            let wt = root.join(format!("judge-{j}"));
5374            if wt.exists() {
5375                let target = fold_target(&home, &root, &wt, isolated);
5376                remove_read_seat_worktree(&mut self.state, &wt).await?;
5377                reclaim_fold_target(&home, target);
5378                self.state.save()?;
5379            }
5380        }
5381        // The design-deliberation stage is finished by the time a tally
5382        // exists — same reasoning as the judges above.
5383        if self.state.config.graph.advise {
5384            for k in 1..=self.state.config.graph.advisors {
5385                let wt = root.join(format!("advisor-{k}"));
5386                if wt.exists() {
5387                    let target = fold_target(&home, &root, &wt, isolated);
5388                    remove_read_seat_worktree(&mut self.state, &wt).await?;
5389                    reclaim_fold_target(&home, target);
5390                    self.state.save()?;
5391                }
5392            }
5393        }
5394        if !folded.is_empty() {
5395            self.state
5396                .event("fold", format!("folded candidates {}", folded.join(", ")));
5397            self.state.save()?;
5398        }
5399        Ok(())
5400    }
5401
5402    // ------------------------------------------------------------ base sync
5403
5404    async fn hold_base_sync(&mut self) -> Result<bool> {
5405        self.hold_base_sync_as(None).await
5406    }
5407
5408    async fn hold_base_sync_kind(
5409        &mut self,
5410        kind: crate::base_sync_recovery::StopKind,
5411    ) -> Result<()> {
5412        self.hold_base_sync_as(Some(kind)).await?;
5413        Ok(())
5414    }
5415
5416    async fn hold_base_sync_as(
5417        &mut self,
5418        kind: Option<crate::base_sync_recovery::StopKind>,
5419    ) -> Result<bool> {
5420        if !self.state.base_sync_blocked() {
5421            self.state.base_sync_recovery.stop = None;
5422            return Ok(false);
5423        }
5424        let head = match self.state.winner() {
5425            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5426                .await
5427                .unwrap_or_default(),
5428            None => String::new(),
5429        };
5430        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5431            && !authorization.consumed
5432            && authorization.stop.head != head
5433        {
5434            authorization.refuse(format!(
5435                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5436                authorization.stop.head
5437            ));
5438        }
5439        self.state.record_base_sync_stop(&head);
5440        if let Some(kind) = kind {
5441            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5442            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5443                stop.kind = kind;
5444            }
5445        }
5446        self.state.save()?;
5447        Ok(true)
5448    }
5449
5450    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5451    /// anything verifies it.
5452    ///
5453    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5454    /// read whatever is checked out in the winner's worktree. Left alone that
5455    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5456    /// it when the run *branched* - and a run takes long enough that the base
5457    /// has usually moved by the time it gets here. A gate that ran there
5458    /// answers "green on the commit this run started from", not "green on
5459    /// what is about to land", and the difference showed up three times in
5460    /// one day as a green run whose merge would have reverted a file another
5461    /// pull request had already landed.
5462    ///
5463    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5464    /// `land::Step::Rebase` calls, rather than a second implementation of the
5465    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5466    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5467    /// doc warns about ("two answers to a question nobody notices until a
5468    /// diff is wrong").
5469    ///
5470    /// Foreign-checkout reviews instead merge the base into the owned shadow
5471    /// worktree, preserving the ancestry required for fast-forward delivery.
5472    /// A conflicting merge is aborted and held for operator resolution.
5473    ///
5474    /// A conflict is not the end of the road: the standing rebase is handed
5475    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5476    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5477    /// land). Once it finishes, review and the gate run as usual on the
5478    /// rebased tree, which is where a breakage the new base caused is caught
5479    /// by the ordinary gate-fix round. magi resolves nothing itself.
5480    ///
5481    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5482    /// the current durable recovery interval, bounds how often the base is
5483    /// synchronized during active recovery (lifetime attempts remain evidence).
5484    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5485    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5486    /// what was tried (rounds spent, paths still conflicted) and the branch
5487    /// and worktree stay exactly as they were - untouched, for a person to
5488    /// look at - which is also what makes re-entering this function
5489    /// afterwards a no-op instead of a second attempt at the same wall. A
5490    /// push failure ends the same way.
5491    async fn sync_to_base(&mut self) -> Result<()> {
5492        if self.state.status == RunStatus::AlreadyInBase {
5493            return Ok(());
5494        }
5495        self.hold_base_sync().await?;
5496        let conflicted = self
5497            .state
5498            .base_sync
5499            .as_ref()
5500            .is_some_and(|s| s.conflict.is_some());
5501        let Some(winner) = self.state.winner().cloned() else {
5502            return Ok(());
5503        };
5504
5505        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5506
5507        let repo = self.state.repo.clone();
5508        let remote = self.state.config.merge.remote.clone();
5509        let base_branch = self.state.base_branch.clone();
5510        let tracking = format!("{remote}/{base_branch}");
5511
5512        let fetched_base =
5513            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5514        // No network, or the remote never had this branch: the run already
5515        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5516        // otherwise), and one that got this far is not blocked by a fetch
5517        // that fails now. It just has no newer tip to compare against.
5518        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5519            return Ok(());
5520        };
5521
5522        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5523        self.state.ensure_base_episode();
5524        let mut pending = self
5525            .state
5526            .base_sync_recovery
5527            .operations
5528            .last()
5529            .filter(|o| !o.finished)
5530            .cloned();
5531        let scratch = self.state.dir().join("base-sync");
5532        if git::rebase_in_progress(&scratch).await {
5533            // Older resource stops predate operation reservations. Import the
5534            // standing Git operation, never retarget it to today's main.
5535            if pending.is_none()
5536                && self.state.base_sync_recovery.operations.is_empty()
5537                && !conflicted
5538                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5539                    .await
5540                    .ok()
5541                    .as_deref()
5542                    == Some(head.as_str())
5543                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5544                && git::is_clean(&winner.worktree).await?
5545            {
5546                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5547                if self.state.base_sync.is_none() {
5548                    self.state.base_sync = Some(BaseSync {
5549                        tip: saved_tip.clone(),
5550                        behind: 0,
5551                        attempts: 0,
5552                        conflict: None,
5553                        already_in: None,
5554                    });
5555                }
5556                self.state.reserve_base_operation(&head, &saved_tip);
5557                self.state.event(
5558                    "base-recovery",
5559                    "imported standing legacy rebase as a spent synchronization reservation",
5560                );
5561                self.state.save()?;
5562                pending = self.state.base_sync_recovery.operations.last().cloned();
5563            }
5564            let expected = pending.as_ref().map(|op| op.tip.as_str());
5565            if expected.is_none()
5566                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5567            {
5568                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5569                if self.state.base_sync.is_none() {
5570                    self.state.base_sync = Some(BaseSync {
5571                        tip: tip.clone(),
5572                        behind: 0,
5573                        attempts: 0,
5574                        conflict: None,
5575                        already_in: None,
5576                    });
5577                }
5578                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5579                self.state.event("land", why);
5580                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5581                    .await?;
5582                return Ok(());
5583            }
5584        }
5585        if let Some(op) = &pending {
5586            // A replay may add the pinned base's commits, never foreign work.
5587            let mut replay_proven =
5588                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5589            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5590                let wait = repository_wait(&self.state, None);
5591                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5592                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5593                    Ok(true) => {
5594                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5595                        self.state.save()?;
5596                        git::worktree_remove(&repo, &scratch).await.ok();
5597                        replay_proven = true;
5598                    }
5599                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5600                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5601                }
5602            }
5603            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5604                if self.state.foreign_checkout.is_some() {
5605                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5606                        .await
5607                        .ok()
5608                        .as_deref()
5609                        == Some(op.head.as_str())
5610                        && git::rev_parse(&repo, &format!("{head}^2"))
5611                            .await
5612                            .ok()
5613                            .as_deref()
5614                            == Some(op.tip.as_str());
5615                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5616                    .await
5617                    .is_ok_and(|m| m.is_empty())
5618                    && let Ok(added) =
5619                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5620                {
5621                    replay_proven = true;
5622                    for commit in added {
5623                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5624                            replay_proven = false;
5625                            break;
5626                        }
5627                    }
5628                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5629                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5630                }
5631            }
5632            // Never retarget an operation across a crash, even if main moved.
5633            // sync_to_head resets and cleans with -x, so even ignored untracked
5634            // files must stop recovery independently of the old tracked tree.
5635            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5636                || (head != op.head
5637                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5638                        .await?
5639                        .is_empty())
5640                || (!git::is_clean(&winner.worktree).await?
5641                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5642                        .await
5643                        .is_ok_and(|o| o.ok()))
5644                || !replay_proven
5645            {
5646                let why = format!(
5647                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5648                    op.id
5649                );
5650                let sync = self
5651                    .state
5652                    .base_sync
5653                    .as_mut()
5654                    .context("reserved synchronization has no lifetime accounting")?;
5655                sync.conflict = Some(why.clone());
5656                self.state.event("land", why);
5657                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5658                    .await?;
5659                return Ok(());
5660            }
5661            if head != op.head {
5662                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5663            }
5664            tip = op.tip.clone();
5665        }
5666        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5667        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5668        if pending.is_none()
5669            && !conflicted
5670            && fetched_base
5671            && self
5672                .state
5673                .base_sync_recovery
5674                .episodes
5675                .last()
5676                .is_some_and(|e| e.boundary.is_some())
5677            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5678            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5679            && owner.execution_runs().last() == Some(&self.state.id)
5680            && owner.status != crate::queue::TaskStatus::Done
5681            && !owner.fresh_start
5682            && owner.review_branch.is_none()
5683            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5684            && let Some(approval) = owner.resumption.as_ref()
5685            && approval.instruction == owner.instruction
5686            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5687            && git::is_clean(&winner.worktree).await?
5688            && self.state.renew_base_episode(&task, approval, &head, &tip)
5689        {
5690            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5691            self.state.save()?;
5692        }
5693
5694        // Before any rebase, and before a recorded conflict is honoured: a
5695        // branch whose change reached the base under other commit ids has
5696        // nothing to rebase and nothing to conflict with, and a run that
5697        // already stopped on that phantom conflict recovers here on resume.
5698        // `behind == 0` with head == tip is a branch the base has since taken
5699        // in whole, whether or not a conflict was ever recorded: the ancestry
5700        // proof must still run (`classify` ignores a head still on the start
5701        // commit).
5702        if (behind > 0 || conflicted || head == tip)
5703            && self
5704                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5705                .await?
5706        {
5707            return Ok(());
5708        }
5709        if conflicted && self.state.base_recovery_pending() {
5710            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5711            let owner =
5712                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5713            let owner_valid = owner.as_ref().is_ok_and(|t| {
5714                t.execution_runs().last() == Some(&self.state.id)
5715                    && t.status != crate::queue::TaskStatus::Done
5716                    && !t.fresh_start
5717                    && t.review_branch.is_none()
5718            });
5719            let invalidation = if !owner_valid {
5720                Some("latest task ownership changed")
5721            } else if !owner.as_ref().is_ok_and(|t| {
5722                authorization
5723                    .generation
5724                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5725            }) {
5726                Some("owner approval generation changed")
5727            } else if authorization.question.as_ref().is_some_and(|question| {
5728                !crate::base_recovery_control::decision_current(
5729                    &crate::queue::Queue::at(crate::run::home().join("queue")),
5730                    &self.state,
5731                    &authorization.task,
5732                    question,
5733                )
5734            }) {
5735                Some("recovery decision ownership, routing or stop evidence changed")
5736            } else if authorization.request.is_some()
5737                && !owner
5738                    .as_ref()
5739                    .is_ok_and(|t| t.operator_decided_at == authorization.decision)
5740            {
5741                Some("owner decision changed")
5742            } else if let Some(scope) = &authorization.scope {
5743                if scope.tip != tip {
5744                    Some("fetched base changed")
5745                } else if !self
5746                    .state
5747                    .base_sync_recovery
5748                    .episodes
5749                    .last()
5750                    .is_some_and(|e| e.id == scope.episode)
5751                {
5752                    Some("active synchronization interval changed")
5753                } else if !owner
5754                    .as_ref()
5755                    .is_ok_and(|t| t.instruction == scope.instruction)
5756                {
5757                    Some("task instruction changed")
5758                } else {
5759                    None
5760                }
5761            } else {
5762                None
5763            };
5764            if let Some(reason) = invalidation {
5765                let authorization = self
5766                    .state
5767                    .base_sync_recovery
5768                    .authorizations
5769                    .last_mut()
5770                    .unwrap();
5771                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5772                self.state.save()?;
5773                return Ok(());
5774            }
5775            if !fetched_base
5776                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5777                || !git::is_clean(&winner.worktree).await?
5778                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5779            {
5780                return Ok(());
5781            }
5782        }
5783        let recovery = conflicted && self.state.consume_base_recovery(&head);
5784        if conflicted {
5785            // Consumption (including a refused changed HEAD) is durable before Git.
5786            self.state.save()?;
5787            if !recovery {
5788                return Ok(());
5789            }
5790        }
5791
5792        if behind == 0 {
5793            // A fixer-finished rebase moves the branch ref before the
5794            // winner's worktree is told (`sync_to_head` below). A run that
5795            // died in between resumes here with `behind == 0` and a tree still
5796            // holding the pre-rebase files, which review and the gate would
5797            // then read. That state is exactly: HEAD moved off the tip the
5798            // rebase started from, yet the tree is still identical to that
5799            // tip. A tree with edits of its own differs from it, so nothing
5800            // is thrown away.
5801            if let Some(from) = self
5802                .state
5803                .rebase_fixes
5804                .iter()
5805                .rev()
5806                .find_map(|r| r.from.clone())
5807                && from != head
5808                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5809                    .await
5810                    .is_ok_and(|o| o.ok())
5811            {
5812                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5813            }
5814            // An earlier attempt may have rebased the branch locally and died
5815            // before pushing it (only the fresh-rebase arm below pushes).
5816            // Publish it now, so the plain push at PR time is not refused as
5817            // a non-fast-forward. A run already holding a recorded conflict
5818            // never reaches here; that case is out of scope.
5819            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5820            if let Some(why) = &conflict {
5821                self.state.status = RunStatus::Blocked;
5822                self.state.event("land", why.clone());
5823            }
5824            self.state.base_sync = Some(BaseSync {
5825                tip,
5826                behind: 0,
5827                attempts,
5828                conflict,
5829                already_in: None,
5830            });
5831            if pending.is_some() {
5832                self.state.finish_base_operation(Some(&head));
5833            }
5834            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5835                .await?;
5836            self.state.save()?;
5837            return Ok(());
5838        }
5839
5840        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5841            let why = format!(
5842                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5843                 synchronization(s); synchronizing again would only race it",
5844                winner.branch
5845            );
5846            self.state.status = RunStatus::Blocked;
5847            self.state.base_sync = Some(BaseSync {
5848                tip,
5849                behind,
5850                attempts,
5851                conflict: Some(why.clone()),
5852                already_in: None,
5853            });
5854            self.state.event("land", why);
5855            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5856                .await?;
5857            self.state.save()?;
5858            return Ok(());
5859        }
5860
5861        if self.state.foreign_checkout.is_some() {
5862            // Delivery to the operator's checkout is fast-forward only. A
5863            // rebase here would discard its saved head from our ancestry.
5864            let wait = repository_wait(&self.state, None);
5865            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5866            if self.state.base_sync.is_none() {
5867                self.state.base_sync = Some(BaseSync {
5868                    tip: tip.clone(),
5869                    behind,
5870                    attempts: 0,
5871                    conflict: None,
5872                    already_in: None,
5873                });
5874            }
5875            self.state.reserve_base_operation(&head, &tip);
5876            self.state.save()?;
5877            let merged = git::git_raw(
5878                &winner.worktree,
5879                &[
5880                    "-c",
5881                    "core.hooksPath=/dev/null",
5882                    "merge",
5883                    "--no-ff",
5884                    "--no-autostash",
5885                    "--no-edit",
5886                    "-m",
5887                    &format!("Merge {tracking} before verifying"),
5888                    &tip,
5889                ],
5890            )
5891            .await?;
5892            let conflict = if merged.ok() {
5893                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5894                self.state.event(
5895                    "land",
5896                    format!("merged {tracking} into {} before verifying", winner.branch),
5897                );
5898                None
5899            } else {
5900                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5901                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5902                }
5903                let why = format!(
5904                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5905                    winner.branch,
5906                    merged.stderr.chars().take(600).collect::<String>()
5907                );
5908                self.state.status = RunStatus::Blocked;
5909                self.state.event("land", &why);
5910                Some(why)
5911            };
5912            let synced = conflict.is_none();
5913            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5914            self.state
5915                .finish_base_operation(synced.then_some(fresh.as_str()));
5916            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5917            self.state.base_sync = Some(BaseSync {
5918                tip,
5919                behind: if synced { 0 } else { behind },
5920                attempts,
5921                conflict,
5922                already_in: None,
5923            });
5924            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5925                .await?;
5926            self.state.save()?;
5927            return Ok(());
5928        }
5929
5930        self.state.event(
5931            "land",
5932            format!(
5933                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5934                winner.branch
5935            ),
5936        );
5937        self.state.save()?;
5938
5939        // The remote's copy of the branch, read now and only if the fetch
5940        // really succeeded (a stale tracking ref must never pin a lease). It is
5941        // pushed over after a rebase only when it is a commit this branch
5942        // already contains, by ancestry or by patch (an earlier rebase of ours
5943        // that never reached the remote): anything else is somebody else's work.
5944        let branch_tracking = format!("{remote}/{}", winner.branch);
5945        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5946        let remote_tip = if self.state.task_pr_branch().is_none()
5947            && matches!(&fetched_branch, Ok(o) if o.ok())
5948        {
5949            git::rev_parse(&repo, &branch_tracking).await.ok()
5950        } else {
5951            None
5952        };
5953        // A remote tip this branch does not contain is somebody else's work:
5954        // rebasing would leave a local tip that can never be pushed. Stop
5955        // before touching anything and say so.
5956        if let Some(theirs) = &remote_tip
5957            && !git::is_ancestor(&repo, theirs, &head).await
5958            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5959                .await
5960                .is_ok_and(|missing| missing.is_empty())
5961        {
5962            let why = format!(
5963                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5964                 them",
5965                short(theirs),
5966                winner.branch
5967            );
5968            self.state.status = RunStatus::Blocked;
5969            self.state.base_sync = Some(BaseSync {
5970                tip,
5971                behind,
5972                attempts,
5973                conflict: Some(why.clone()),
5974                already_in: None,
5975            });
5976            self.state.event("land", why);
5977            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5978                .await?;
5979            self.state.save()?;
5980            return Ok(());
5981        }
5982
5983        if self.state.base_sync.is_none() {
5984            self.state.base_sync = Some(BaseSync {
5985                tip: tip.clone(),
5986                behind,
5987                attempts: 0,
5988                conflict: None,
5989                already_in: None,
5990            });
5991        }
5992        let scratch = self.state.dir().join("base-sync");
5993        if let Err(error) = crate::rebase::prepare(&mut self.state, &scratch, &winner.branch).await
5994        {
5995            if error.is::<crate::repo_lock::ResourceBlocked>() {
5996                return Ok(());
5997            }
5998            return Err(error);
5999        }
6000        self.state.reserve_base_operation(&head, &tip);
6001        self.state.save()?;
6002        let rebased =
6003            match crate::rebase::rebase_prepared(&mut self.state, &scratch, &winner.branch, &tip)
6004                .await
6005            {
6006                Ok(crate::rebase::Rebased::Applied) => Ok(None),
6007                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
6008                Err(e) => Err(e),
6009            };
6010        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
6011        let stop_kind = match &rebased {
6012            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
6013            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
6014            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
6015        };
6016        match rebased {
6017            Ok(None) => {
6018                // The branch ref moved, but a worktree that already had it
6019                // checked out (the winner's) was not told; sync its index and
6020                // files before anything reads them.
6021                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
6022                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
6023                self.state.finish_base_operation(Some(&fresh));
6024                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
6025                let mut conflict = None;
6026                if let Some(pinned) = &remote_tip {
6027                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
6028                    match pushed {
6029                        Ok(o) if o.ok() => self.state.event(
6030                            "land",
6031                            format!("pushed rebased {} to {remote}", winner.branch),
6032                        ),
6033                        Ok(o) => {
6034                            conflict = Some(format!(
6035                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
6036                                winner.branch,
6037                                short(pinned),
6038                                o.stderr.chars().take(600).collect::<String>()
6039                            ));
6040                        }
6041                        Err(e) => {
6042                            conflict = Some(format!(
6043                                "rebased {} locally but could not push it: {e:#}",
6044                                winner.branch
6045                            ));
6046                        }
6047                    }
6048                }
6049                if let Some(why) = &conflict {
6050                    self.state.status = RunStatus::Blocked;
6051                    self.state.event("land", why.clone());
6052                }
6053                self.state.base_sync = Some(BaseSync {
6054                    tip: tip.clone(),
6055                    behind: 0,
6056                    attempts,
6057                    conflict,
6058                    already_in: None,
6059                });
6060                self.state
6061                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
6062            }
6063            Ok(Some(conflict)) => {
6064                self.state.finish_base_operation(None);
6065                let why = format!(
6066                    "{} conflicts with {tracking} and did not rebase: {}",
6067                    winner.branch,
6068                    conflict.chars().take(600).collect::<String>()
6069                );
6070                self.state.status = RunStatus::Blocked;
6071                self.state.base_sync = Some(BaseSync {
6072                    tip,
6073                    behind,
6074                    attempts,
6075                    conflict: Some(why.clone()),
6076                    already_in: None,
6077                });
6078                self.state.event("land", why);
6079            }
6080            // A refused Git write or a busy repository: nothing is wrong with
6081            // the branch. The scratch rebase stays standing and the run
6082            // resumes into it (`base_sync` must not claim a conflict).
6083            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
6084                self.state.save()?;
6085                return Ok(());
6086            }
6087            Err(e) => {
6088                self.state.finish_base_operation(None);
6089                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
6090                self.state.status = RunStatus::Blocked;
6091                self.state.base_sync = Some(BaseSync {
6092                    tip,
6093                    behind,
6094                    attempts,
6095                    conflict: Some(why.clone()),
6096                    already_in: None,
6097                });
6098                self.state.event("land", why);
6099            }
6100        }
6101        if !self.state.base_sync_blocked() {
6102            self.state.base_sync_recovery.stop = None;
6103        }
6104        self.hold_base_sync_kind(stop_kind).await?;
6105        self.state.save()?;
6106        Ok(())
6107    }
6108
6109    /// Push a branch an earlier attempt rebased locally but never published,
6110    /// pinned to the remote tip read right after a successful fetch. Returns
6111    /// the reason when the run must stop; `None` when there was nothing to do
6112    /// (no remote copy, the same tip, or a remote copy this branch already
6113    /// contains, which the PR-time push fast-forwards) or the push succeeded.
6114    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
6115        // Task competitions publish only after the PR text gate, to their
6116        // public head; never resurrect a candidate branch on the remote.
6117        if self.state.task_pr_branch().is_some() {
6118            return None;
6119        }
6120        let repo = self.state.repo.clone();
6121        let remote = self.state.config.merge.remote.clone();
6122        let fetched = git::fetch(&repo, &remote, branch).await;
6123        if !matches!(&fetched, Ok(o) if o.ok()) {
6124            return None;
6125        }
6126        let branch_tracking = format!("{remote}/{branch}");
6127        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
6128        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
6129            return None;
6130        }
6131        if !crate::reconcile::origin_missing(&repo, head, &theirs)
6132            .await
6133            .is_ok_and(|missing| missing.is_empty())
6134        {
6135            return Some(format!(
6136                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
6137                 them",
6138                short(&theirs)
6139            ));
6140        }
6141        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6142            Ok(o) if o.ok() => {
6143                self.state
6144                    .event("land", format!("pushed rebased {branch} to {remote}"));
6145                None
6146            }
6147            Ok(o) => Some(format!(
6148                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6149                 someone may have pushed): {}",
6150                short(&theirs),
6151                o.stderr.chars().take(600).collect::<String>()
6152            )),
6153            Err(e) => Some(format!(
6154                "{branch} is rebased locally but could not be pushed: {e:#}"
6155            )),
6156        }
6157    }
6158
6159    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6160    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6161    /// whether it did.
6162    ///
6163    /// Checked only when the base is ahead of the branch. A failing check is
6164    /// "not proven" - the ordinary rebase path then decides - never a reason to
6165    /// stop the run.
6166    ///
6167    /// The remote copy of the branch is held to the same standard as the local
6168    /// one: if it carries a tip this worktree does not, that tip must itself be
6169    /// proven in the base, or nothing is settled (a pull request would
6170    /// otherwise be closed over commits nobody checked). The pull request is
6171    /// closed *before* the terminal status is saved; if that fails for a
6172    /// reason other than a refusal (no network, a `gh` error) the run is left
6173    /// `Blocked` with the reason as its conflict, which a resume retries -
6174    /// the same recovery a phantom conflict gets.
6175    async fn settle_already_in(
6176        &mut self,
6177        branch: &str,
6178        tip: &str,
6179        head: &str,
6180        attempts: usize,
6181        behind: usize,
6182    ) -> Result<bool> {
6183        let repo = self.state.repo.clone();
6184        let remote = self.state.config.merge.remote.clone();
6185        let start = self.state.base_commit.clone();
6186        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6187            Ok(Some(e)) => e,
6188            Ok(None) => return Ok(false),
6189            Err(e) => {
6190                tracing::warn!("already-in-base check for {branch}: {e:#}");
6191                return Ok(false);
6192            }
6193        };
6194        let mut verified = vec![head.to_owned()];
6195        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6196        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6197        if matches!(&fetched, Ok(o) if o.ok())
6198            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6199            && theirs != head
6200        {
6201            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6202                Ok(Some(_)) => verified.push(theirs),
6203                _ => return Ok(false),
6204            }
6205        }
6206        let base_branch = self.state.base_branch.clone();
6207        let message = format!(
6208            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6209             land",
6210            evidence.names(),
6211            evidence.proof.as_str()
6212        );
6213        let closed =
6214            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6215                .await;
6216        match closed {
6217            Ok(Ok(url)) => self
6218                .state
6219                .event("land", format!("closed {url}: superseded on {base_branch}")),
6220            Ok(Err(why)) => self
6221                .state
6222                .event("land", format!("did not close a pull request: {why}")),
6223            Err(e) => {
6224                let why = format!(
6225                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6226                     not be closed ({e:#}); resume to retry"
6227                );
6228                self.state.status = RunStatus::Blocked;
6229                self.state.base_sync = Some(BaseSync {
6230                    tip: tip.to_owned(),
6231                    behind,
6232                    attempts,
6233                    conflict: Some(why.clone()),
6234                    already_in: None,
6235                });
6236                self.state.event("land", why);
6237                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6238                    .await?;
6239                self.state.save()?;
6240                return Ok(true);
6241            }
6242        }
6243        self.state.status = RunStatus::AlreadyInBase;
6244        self.state.base_sync_recovery.stop = None;
6245        self.state.base_sync = Some(BaseSync {
6246            tip: tip.to_owned(),
6247            behind,
6248            attempts,
6249            conflict: None,
6250            already_in: Some(evidence),
6251        });
6252        self.state.event("land", message);
6253        self.state.save()?;
6254        self.settle_questions();
6255        Ok(true)
6256    }
6257
6258    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6259    /// last landed the winner on, once it has run, else the commit the run
6260    /// branched from.
6261    ///
6262    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6263    /// and `vote` all happen before there is a winner to rebase, so they
6264    /// compare every candidate against the branch point on purpose, and a
6265    /// base that moves after they are already done cannot change an answer
6266    /// they already gave.
6267    fn landing_base(&self) -> String {
6268        self.state
6269            .base_sync
6270            .as_ref()
6271            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6272    }
6273
6274    // ------------------------------------------------------- operator fix
6275
6276    /// Route specific, already-recorded review findings to a fixer for a
6277    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6278    /// entry point.
6279    ///
6280    /// Distinct from `review_loop`'s own fix step in three ways: it never
6281    /// runs a reviewer wave, it never spends review-round budget, and what
6282    /// happened is recorded as an [`OperatorFixRequest`] appended to
6283    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6284    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6285    /// and vote must never be rewritten to look like a manufactured blocking
6286    /// verdict.
6287    ///
6288    /// Only meaningful once review has actually concluded: `Ready` (handed
6289    /// off with findings still open, or simply concluded clean while minor
6290    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6291    /// gate failed). Everything else is refused: a run still in progress
6292    /// should simply be resumed, and a `Merged` run's branch has already
6293    /// landed — reopening *this* run's own record cannot change that, so the
6294    /// answer there is a fresh `magi review <branch>`.
6295    ///
6296    /// A real commit here re-verifies through a fresh, ordinary review-only
6297    /// run on the same branch ([`Self::review`]) rather than reopening this
6298    /// run's own `review_loop`: once any round in this run's history went
6299    /// clean, `review_conclusion` treats that as permanent by design (the
6300    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6301    /// way to force one more genuine reviewer wave out of *this* run without
6302    /// either rewriting history or weakening that guarantee for every other
6303    /// caller. A review-only run costs nothing extra — no implementation, no
6304    /// judging, no vote — and exercises the exact same review → verify →
6305    /// gate → (human) merge path, unmodified.
6306    pub async fn fix_selected(
6307        &mut self,
6308        ids: &[String],
6309        reason: &str,
6310        allow_stale: bool,
6311    ) -> Result<()> {
6312        let reason = reason.trim();
6313        if reason.is_empty() {
6314            bail!("a fix request needs a reason — that is the operator's own record of why");
6315        }
6316        if ids.is_empty() {
6317            bail!("no finding id given");
6318        }
6319        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6320            bail!(
6321                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6322                 has already concluded — can be given a targeted fix. A run still \
6323                 in progress should simply be resumed; a `merged` run's branch has \
6324                 already landed, so its answer is a fresh `magi review <branch>`, \
6325                 not reopening this run's own record",
6326                self.state.id,
6327                self.state.status.as_str()
6328            );
6329        }
6330        // Reassignment shares this claim through its atomic task save. Check
6331        // the canonical run identity inside it, before preflight or any await.
6332        let _claim = FixClaim::acquire(&self.state.dir())?;
6333        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6334        let Some(winner) = self.state.winner().cloned() else {
6335            bail!("run {} has no winning candidate to fix", self.state.id);
6336        };
6337        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6338            bail!(
6339                "branch `{}` no longer exists; this run cannot be extended",
6340                winner.branch
6341            );
6342        }
6343        let home = crate::run::home();
6344        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6345            bail!(
6346                "run {} is currently being worked on by another magi process",
6347                self.state.id
6348            );
6349        }
6350        // Resolve every id before spending anything — an unknown id refuses
6351        // the whole request rather than silently dropping it — and dedup
6352        // while keeping the operator's own order.
6353        let mut seen = BTreeSet::new();
6354        let mut findings = Vec::new();
6355        let mut missing = Vec::new();
6356        for id in ids {
6357            if !seen.insert(id.clone()) {
6358                continue;
6359            }
6360            match self.state.finding(id) {
6361                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6362                    id: f.id.clone(),
6363                    severity: f.severity,
6364                    reviewer_vote: rec.vote,
6365                    round: round.round,
6366                    round_head: round.head.clone(),
6367                    reviewer: rec.reviewer,
6368                    agent: rec.agent.clone(),
6369                    file: f.file.clone(),
6370                    line: f.line,
6371                    title: f.title.clone(),
6372                    detail: f.detail.clone(),
6373                    outcome: OperatorFixOutcome::Pending,
6374                }),
6375                None => missing.push(id.clone()),
6376            }
6377        }
6378        if !missing.is_empty() {
6379            bail!(
6380                "unknown finding id(s): {}; nothing was changed",
6381                missing.join(", ")
6382            );
6383        }
6384
6385        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6386        let stale_details: Vec<(String, String)> = findings
6387            .iter()
6388            .filter(|f| f.round_head != head_at_request)
6389            .map(|f| (f.id.clone(), f.round_head.clone()))
6390            .collect();
6391        let stale = !stale_details.is_empty();
6392        if stale && !allow_stale {
6393            bail!(
6394                "the branch has moved since some finding(s) were raised — {} — now \
6395                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6396                stale_details
6397                    .iter()
6398                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6399                    .collect::<Vec<_>>()
6400                    .join(", "),
6401                short(&head_at_request)
6402            );
6403        }
6404
6405        let request = OperatorFixRequest {
6406            requested_at: Timestamp::now(),
6407            reason: reason.to_owned(),
6408            findings,
6409            head_at_request: head_at_request.clone(),
6410            allow_stale,
6411            stale,
6412            fix: None,
6413            result_head: None,
6414            follow_up_review_run: None,
6415        };
6416        self.state.event(
6417            "fix",
6418            format!(
6419                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6420                request.findings.len(),
6421                request
6422                    .findings
6423                    .iter()
6424                    .map(|f| f.id.as_str())
6425                    .collect::<Vec<_>>()
6426                    .join(", "),
6427            ),
6428        );
6429        // Recorded now, before any worktree work or the fixer call itself —
6430        // and re-saved at each checkpoint below: a crash at any point after
6431        // this (mid fixer call, mid follow-up review) must not lose the fact
6432        // that this was requested, for which findings, and why. Everything
6433        // past this point reads and writes through `request_index` rather
6434        // than a local variable, since `request` itself is moved here.
6435        self.state.operator_fixes.push(request);
6436        self.state.save()?;
6437        let request_index = self.state.operator_fixes.len() - 1;
6438
6439        // A fresh, dedicated worktree for this one call, never the winner's
6440        // own worktree in place: that one may already be gone (folded away),
6441        // and reusing it in place would leave the branch checked out there
6442        // when the follow-up review below tries to check it out again. Freed
6443        // immediately after, either way — but only once confirmed clean:
6444        // `worktree_remove` is a `git worktree remove --force`, which would
6445        // otherwise discard uncommitted work left there by the operator or
6446        // another process before this had a chance to even look at it.
6447        let repo = self.state.repo.clone();
6448        let coordination = self.coordinate(&repo).await?;
6449        if winner.worktree.exists() {
6450            // Lockfiles a rescue commit withheld stay untracked on purpose and
6451            // are already recorded; they are not the operator's work to protect.
6452            let dirty = git::git(
6453                &winner.worktree,
6454                &["status", "--porcelain", "--untracked-files=all"],
6455            )
6456            .await?;
6457            let only_withheld = dirty.lines().all(|l| {
6458                l.strip_prefix("?? ")
6459                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6460            });
6461            if !only_withheld {
6462                bail!(
6463                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6464                     discard them first",
6465                    winner.worktree.display()
6466                );
6467            }
6468            git::worktree_remove(&self.state.repo, &winner.worktree)
6469                .await
6470                .ok();
6471        }
6472        let fix_worktree = self.state.worktree_root().join("operator-fix");
6473        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6474        git::git(
6475            &self.state.repo,
6476            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6477        )
6478        .await
6479        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6480        if !git::is_clean(&fix_worktree).await? {
6481            git::worktree_remove(&self.state.repo, &fix_worktree)
6482                .await
6483                .ok();
6484            bail!(
6485                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6486                winner.branch
6487            );
6488        }
6489        let repo_for_setup = self.state.repo.clone();
6490        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6491            Ok(()) => {
6492                worktree_setup::prepare_with(
6493                    &self.state.config,
6494                    &repo_for_setup,
6495                    &fix_worktree,
6496                    crate::build_target::Coordination::Held(coordination.as_ref()),
6497                )
6498                .await
6499            }
6500            Err(e) => Err(e),
6501        };
6502        if let Err(e) = setup {
6503            git::worktree_remove(&self.state.repo, &fix_worktree)
6504                .await
6505                .ok();
6506            return Err(e);
6507        }
6508
6509        // This explicit operator request retries availability independently
6510        // of execute(); the previous call has already saved and cleaned up.
6511        self.state.resource_block = None;
6512
6513        let run_id = self.state.id.clone();
6514        let prompts = self.state.config.prompts.clone();
6515        let language = self.state.config.graph.language.clone();
6516        let sessions = self.state.config.graph.sessions;
6517        let artifacts = agent::artifacts_dir(&self.state.dir());
6518        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6519            .findings
6520            .iter()
6521            .map(|f| Finding {
6522                id: f.id.clone(),
6523                severity: f.severity,
6524                file: f.file.clone(),
6525                line: f.line,
6526                title: f.title.clone(),
6527                detail: f.detail.clone(),
6528            })
6529            .collect();
6530        let fix_prompt = prompt::operator_fix_in(
6531            &self.state.instruction,
6532            &finding_list,
6533            reason,
6534            &stale_details,
6535            &head_at_request,
6536            &language,
6537            &self.state.config.graph.github_language,
6538        );
6539        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6540        drop(coordination);
6541        let (job, seat, out) = self
6542            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6543                prompt: fix_prompt.clone(),
6544                spec,
6545                seat,
6546                cwd: fix_worktree.clone(),
6547                timeout,
6548                allow_write: true,
6549                sessions,
6550                artifacts: artifacts.clone(),
6551                stem: "operator-fix".to_owned(),
6552                handover: None,
6553            })
6554            .await;
6555        let agent_id = seat.agent.clone();
6556
6557        let mut fix = FixRecord {
6558            agent: agent_id,
6559            addressed: Vec::new(),
6560            rejected: Vec::new(),
6561            notes: String::new(),
6562            committed: false,
6563            failed: None,
6564            duration_ms: 0,
6565            continuation: None,
6566        };
6567        let mut final_seat = seat.clone();
6568        match out {
6569            AgentOutcome::Ok(o) => {
6570                fix.duration_ms = o.duration_ms;
6571                let parsed = verdict::extract_json::<FixReport>(&o.text);
6572                let incomplete_reason = match &parsed {
6573                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6574                        "the reply parsed, but it reported a command whose own CLI \
6575                         never confirmed an exit status"
6576                            .to_owned(),
6577                    ),
6578                    Ok(_) => None,
6579                    Err(e) => Some(e.to_string()),
6580                };
6581                match incomplete_reason {
6582                    None => {
6583                        let report = parsed.expect("checked Ok above");
6584                        fix.addressed = report.addressed;
6585                        fix.rejected = report.rejected;
6586                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6587                    }
6588                    Some(reason) => {
6589                        let (resumed_seat, resolved, failure, cont) = self
6590                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6591                            .await;
6592                        fix.duration_ms += cont.cumulative_wait_ms;
6593                        fix.continuation = Some(cont);
6594                        final_seat = resumed_seat;
6595                        match resolved {
6596                            Some(report) => {
6597                                fix.addressed = report.addressed;
6598                                fix.rejected = report.rejected;
6599                                fix.notes =
6600                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6601                            }
6602                            None => fix.failed = failure,
6603                        }
6604                    }
6605                }
6606            }
6607            AgentOutcome::Dropped(o) => {
6608                fix.duration_ms = o.duration_ms;
6609                let why = o
6610                    .dropped
6611                    .as_ref()
6612                    .map(|d| d.why.as_str())
6613                    .unwrap_or("the CLI ended the stream without delivering its answer");
6614                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6615            }
6616            AgentOutcome::Quota(o) => {
6617                self.state.quota.push(QuotaLoss::from_report(
6618                    final_seat.key.clone(),
6619                    "fix",
6620                    o.quota.as_ref(),
6621                ));
6622                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6623            }
6624            AgentOutcome::Failed(e) => fix.failed = Some(e),
6625            AgentOutcome::ResourceBlocked(e) => {
6626                fix.failed = Some(format!("the fixer was not started: {e}"));
6627            }
6628        }
6629        // Even a deferred call must rescue edits, release the worktree and
6630        // finalize its record below so an explicit `magi fix` can retry.
6631        if fix.continuation.is_none() {
6632            fix.continuation = Some(ContinuationRecord::not_needed());
6633        }
6634        self.state.seats.insert(final_seat.key.clone(), final_seat);
6635
6636        let rescue_message = format!(
6637            "magi: operator-selected fix ({}) (uncommitted work)",
6638            self.state.operator_fixes[request_index]
6639                .findings
6640                .iter()
6641                .map(|f| f.id.as_str())
6642                .collect::<Vec<_>>()
6643                .join(", ")
6644        );
6645        let repo = self.state.repo.clone();
6646        let coordination = self.coordinate(&repo).await?;
6647        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6648            self.state.note_withheld("fix", &r.withheld);
6649        }
6650        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6651        fix.committed = after != head_at_request;
6652        git::worktree_remove(&self.state.repo, &fix_worktree)
6653            .await
6654            .ok();
6655        drop(coordination);
6656
6657        self.state.event(
6658            "fix",
6659            match &fix.failed {
6660                Some(reason) => format!(
6661                    "operator fix: adoption report was lost ({reason}); {}",
6662                    if fix.committed {
6663                        "committed"
6664                    } else {
6665                        "NO new commit"
6666                    }
6667                ),
6668                None => format!(
6669                    "operator fix: {} addressed, {} rejected, {}",
6670                    fix.addressed.len(),
6671                    fix.rejected.len(),
6672                    if fix.committed {
6673                        "committed"
6674                    } else {
6675                        "NO new commit"
6676                    }
6677                ),
6678            },
6679        );
6680
6681        // Every selected finding gets an outcome — never left `Pending` once
6682        // the fixer's own turn is over. A report that never came back at all
6683        // marks every one of them `Unreported`, not silently "not addressed":
6684        // quota, a dropped stream, or an exhausted continuation are gaps in
6685        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6686        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6687        for f in &mut self.state.operator_fixes[request_index].findings {
6688            f.outcome = if fix.failed.is_some() {
6689                OperatorFixOutcome::Unreported
6690            } else if fix.addressed.contains(&f.id) {
6691                OperatorFixOutcome::Addressed
6692            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6693                OperatorFixOutcome::Rejected { why: r.why.clone() }
6694            } else {
6695                OperatorFixOutcome::Unreported
6696            };
6697        }
6698
6699        let committed = fix.committed;
6700        if committed {
6701            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6702        }
6703        self.state.operator_fixes[request_index].fix = Some(fix);
6704        // Saved again now that the fixer's own outcome is final, on top of
6705        // the save right after the request was first pushed above.
6706        self.state.save()?;
6707        if self.state.resource_block.is_some() {
6708            return self.stop_for_resource();
6709        }
6710
6711        if committed {
6712            self.state.event(
6713                "fix",
6714                format!(
6715                    "operator fix committed {}; opening a follow-up review-only run",
6716                    short(&after)
6717                ),
6718            );
6719            // The operator asked for the fix, and the follow-up serves whatever
6720            // task the run it follows served.
6721            let origin =
6722                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6723            match Self::review(
6724                &self.state.repo,
6725                &winner.branch,
6726                self.state.config.clone(),
6727                origin,
6728            )
6729            .await
6730            {
6731                Ok(mut follow_up) => {
6732                    follow_up.state.event(
6733                        "start",
6734                        format!(
6735                            "requested by an operator fix on run {} for finding(s) {}",
6736                            self.state.id,
6737                            self.state.operator_fixes[request_index]
6738                                .findings
6739                                .iter()
6740                                .map(|f| f.id.as_str())
6741                                .collect::<Vec<_>>()
6742                                .join(", "),
6743                        ),
6744                    );
6745                    follow_up.state.save()?;
6746                    let follow_up_id = follow_up.state.id.clone();
6747                    // The follow-up is a run like any other: it belongs to the
6748                    // task of the run it follows, or to one filed for it.
6749                    let adopted = match crate::direct::adopt(
6750                        &follow_up.state,
6751                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6752                    ) {
6753                        Ok(a) => a,
6754                        Err(e) => {
6755                            // An ownerless run must not spend agent calls; it
6756                            // stays saved, and `magi run --resume` adopts it.
6757                            self.state.event(
6758                                "fix",
6759                                format!(
6760                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6761                                ),
6762                            );
6763                            self.state.save()?;
6764                            return Ok(());
6765                        }
6766                    };
6767                    let executed = follow_up.execute().await;
6768                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6769                    if let Some(a) = adopted {
6770                        a.finish(&follow_up.state, executed);
6771                    }
6772                    if let Some(e) = failure {
6773                        self.state.event(
6774                            "fix",
6775                            format!(
6776                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6777                            ),
6778                        );
6779                    }
6780                    self.state.operator_fixes[request_index].follow_up_review_run =
6781                        Some(follow_up_id);
6782                }
6783                Err(e) => {
6784                    self.state.event(
6785                        "fix",
6786                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6787                    );
6788                }
6789            }
6790            self.state.save()?;
6791        }
6792
6793        Ok(())
6794    }
6795
6796    // --------------------------------------------------------------- review
6797
6798    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6799    /// names, in order, each at most once, moving on only when the call
6800    /// advances (an error, a quota hit or nothing usable - the same decision
6801    /// point `agent::chain_advances` is for the other chained roles).
6802    ///
6803    /// Each agent gets its own seat from `Runner::seat`: the same agent
6804    /// continues its conversation (the winner's own implementer seat when it
6805    /// is the winner's author, now that the competition is over), another
6806    /// takes a fresh one so the full prompt is sent again. A handover is
6807    /// recorded under `node`, which is what makes the fallback stick for the
6808    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6809    /// is returned, so an exhausted chain reads exactly like a single failed
6810    /// fixer: one quota loss, the same wording, the same refund. An earlier
6811    /// attempt's edits are left in the tree and judged, with the final
6812    /// attempt's, by what git says afterwards.
6813    ///
6814    /// Returns the job that produced the outcome, for `continue_fix_report`.
6815    async fn ask_fixer(
6816        &mut self,
6817        winner: &Candidate,
6818        node: &'static str,
6819        round: Option<usize>,
6820        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6821    ) -> (SeatJob, SeatState, AgentOutcome) {
6822        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6823        self.ask_fixer_among(attempts, node, round, None, build)
6824            .await
6825    }
6826
6827    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6828    /// worktree and a commit: before the next agent is asked after one that
6829    /// did not satisfy the call, the worktree is put back on that commit, so
6830    /// a failed agent's partial edits are not carried into the next one.
6831    async fn ask_fixer_among(
6832        &mut self,
6833        attempts: Vec<(AgentSpec, String)>,
6834        node: &'static str,
6835        round: Option<usize>,
6836        restore: Option<(&Path, &str)>,
6837        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6838    ) -> (SeatJob, SeatState, AgentOutcome) {
6839        let run_id = self.state.id.clone();
6840        let prompts = self.state.config.prompts.clone();
6841        let cache = self.state.config.cache_dir();
6842        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6843        let mut last = None;
6844        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6845            let seat = self.seat(&key, &spec.id);
6846            let mut job = build(spec.clone(), seat);
6847            if i > 0 {
6848                job.stem = format!("{}-{}", job.stem, spec.id);
6849            }
6850            let ctx = WaveCtx {
6851                carry_seats: false,
6852                pauses: Some((&self.pause, &self.interrupt)),
6853                run: &run_id,
6854                node,
6855                prompts: &prompts,
6856                cache: cache.as_deref(),
6857                round,
6858            };
6859            let (seat, out) =
6860                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6861            // A resource block never advances the chain: the chosen agent
6862            // was not asked, so it must be the one asked again on resume.
6863            let advances = match &out {
6864                AgentOutcome::Ok(o) => agent::output_advances(o),
6865                AgentOutcome::ResourceBlocked(_) => false,
6866                _ => true,
6867            };
6868            if let Some(next) = ids.get(i + 1)
6869                && advances
6870            {
6871                self.state.seats.insert(seat.key.clone(), seat.clone());
6872                let class =
6873                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6874                let reason = match &out {
6875                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6876                    other => fail_reason(other),
6877                };
6878                // A failed agent's edits must not reach the next one, and a
6879                // restore that did not work ends the chain: the call fails
6880                // and the caller's own (confirmed) restore decides.
6881                if let Some((wt, rev)) = restore
6882                    && let Err(e) = self.graft_restore(wt, rev).await
6883                {
6884                    let why =
6885                        format!("could not restore the worktree before the next fixer: {e:#}");
6886                    tracing::warn!("{why}");
6887                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6888                        AgentOutcome::ResourceBlocked(why)
6889                    } else {
6890                        AgentOutcome::Failed(why)
6891                    };
6892                    last = Some((job, seat, out));
6893                    break;
6894                }
6895                record_handover(
6896                    &mut self.state,
6897                    node,
6898                    &seat.key,
6899                    &spec.id,
6900                    next,
6901                    &class,
6902                    &reason,
6903                );
6904                continue;
6905            }
6906            last = Some((job, seat, out));
6907            break;
6908        }
6909        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6910        if let AgentOutcome::Quota(output) = &out
6911            && node != "graft"
6912        {
6913            // Graft is best effort: its caller records the loss and restores
6914            // the pre-graft tree, then proceeds to review and gate.
6915            // Every fallback was tried. Machine outages cannot be repaired by
6916            // spending a fix round; use each caller's resumable resource path.
6917            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6918            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6919            self.state.quota.push(loss);
6920            self.state.seats.insert(seat.key.clone(), seat.clone());
6921            self.state.status = RunStatus::Blocked;
6922            return (
6923                job,
6924                seat,
6925                AgentOutcome::ResourceBlocked(
6926                    "fixer unavailable; restore agent availability and resume".into(),
6927                ),
6928            );
6929        }
6930        (job, seat, out)
6931    }
6932
6933    /// The one repair a verification-only failure earns (see
6934    /// [`crate::recovery`]), before the first review round of a run that took
6935    /// a branch over.
6936    ///
6937    /// Saved as started *before* the fixer is called, so a resume never
6938    /// repairs twice; a start found on re-entry is rolled back to the commit
6939    /// it began from. A failed or unusable fixer leaves the tree as it was and
6940    /// the review goes on over it, so the first round's own verification
6941    /// reports the failure again. The failure is never carried into a round as
6942    /// a result: verification is re-run on whatever commit this leaves.
6943    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6944        use crate::recovery::{Mode, Repair};
6945        if !self.state.reviews.is_empty() {
6946            return Ok(());
6947        }
6948        let Some(mut rec) = self.state.recovery.clone() else {
6949            return Ok(());
6950        };
6951        if rec.mode != Mode::VerifyRepair {
6952            return Ok(());
6953        }
6954        let wt = winner.worktree.clone();
6955        let mut head = git::rev_parse(&wt, "HEAD").await?;
6956        match rec.repair.clone() {
6957            Repair::Done { .. } => return Ok(()),
6958            Repair::WaitingForRestore { before } => {
6959                self.graft_restore(&wt, &before).await?;
6960                head = before;
6961                rec.repair = Repair::NotStarted;
6962                self.state.recovery = Some(rec.clone());
6963                self.state.save()?;
6964            }
6965            Repair::Started { before } => {
6966                // Interrupted: the unfinished edits are not trusted.
6967                // An unconfirmed rollback blocks the run with `Started` kept,
6968                // so a resume retries it; the tree is never reviewed as is.
6969                if let Err(e) = self.graft_restore(&wt, &before).await {
6970                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6971                        return Err(e);
6972                    }
6973                    return self
6974                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6975                        .await;
6976                }
6977                rec.repair = Repair::Done {
6978                    before: before.clone(),
6979                    after: before,
6980                    committed: false,
6981                    failed: Some("interrupted; rolled back".to_owned()),
6982                };
6983                self.state
6984                    .event("recovery", "the repair was interrupted and rolled back");
6985                self.state.recovery = Some(rec);
6986                self.state.save()?;
6987                return Ok(());
6988            }
6989            Repair::NotStarted => {}
6990        }
6991        if crate::recovery::recheck(&mut rec, &head) {
6992            self.state.event("recovery", rec.summary());
6993            self.state.recovery = Some(rec);
6994            self.state.save()?;
6995            return Ok(());
6996        }
6997        rec.repair = Repair::Started {
6998            before: head.clone(),
6999        };
7000        self.state.recovery = Some(rec.clone());
7001        self.state.save()?;
7002
7003        let language = self.state.config.graph.language.clone();
7004        let sessions = self.state.config.graph.sessions;
7005        let artifacts = agent::artifacts_dir(&self.state.dir());
7006        let fix_prompt = prompt::recovery_fix(
7007            &self.state.instruction,
7008            &rec,
7009            self.state.config.graph.review_rounds,
7010            &language,
7011        );
7012        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7013        self.state.event(
7014            "recovery",
7015            format!(
7016                "repair: the fixer starts from the failed verification on {}",
7017                short(&head)
7018            ),
7019        );
7020        let (_job, seat, out) = self
7021            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
7022                prompt: fix_prompt.clone(),
7023                spec,
7024                seat,
7025                cwd: wt.clone(),
7026                timeout,
7027                allow_write: true,
7028                sessions,
7029                artifacts: artifacts.clone(),
7030                stem: "fix-recovery".to_owned(),
7031                handover: None,
7032            })
7033            .await;
7034        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7035            rec.repair = Repair::WaitingForRestore { before: head };
7036            self.state.recovery = Some(rec);
7037            return self.stop_for_resource();
7038        }
7039        let failed = match &out {
7040            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
7041            AgentOutcome::Quota(o) => {
7042                self.state.quota.push(QuotaLoss::from_report(
7043                    seat.key.clone(),
7044                    "fix",
7045                    o.quota.as_ref(),
7046                ));
7047                Some("rate limited (quota); the fixer could not run".to_owned())
7048            }
7049            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
7050            AgentOutcome::Failed(e) => Some(e.clone()),
7051        };
7052        self.state.seats.insert(seat.key.clone(), seat);
7053        let coordination = self.coordinate(&wt).await?;
7054        let failed = match failed {
7055            // The agent's partial edits are not kept.
7056            Some(why) => {
7057                if let Err(e) = git::restore_to(&wt, &head).await {
7058                    return self
7059                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
7060                        .await;
7061                }
7062                Some(why)
7063            }
7064            None => {
7065                if let Ok(r) = git::rescue_commit(
7066                    &wt,
7067                    "magi: recovery repair of the failed verification (uncommitted work)",
7068                )
7069                .await
7070                {
7071                    self.state.note_withheld("fix", &r.withheld);
7072                }
7073                None
7074            }
7075        };
7076        let after = git::rev_parse(&wt, "HEAD").await?;
7077        drop(coordination);
7078        self.state.event(
7079            "recovery",
7080            match &failed {
7081                Some(why) => format!("repair: nothing kept ({why})"),
7082                None if after != head => {
7083                    format!("repair: committed {}", short(&after))
7084                }
7085                None => "repair: the fixer made no commit".to_owned(),
7086            },
7087        );
7088        rec.repair = Repair::Done {
7089            before: head.clone(),
7090            committed: after != head,
7091            after,
7092            failed,
7093        };
7094        self.state.recovery = Some(rec);
7095        self.state.save()?;
7096        Ok(())
7097    }
7098
7099    /// A repair whose rollback could not be confirmed: the run is blocked and
7100    /// resumable, and the unconfirmed state is never reviewed.
7101    async fn stop_recovery(
7102        &mut self,
7103        mut rec: crate::recovery::RecoveryRecord,
7104        head: &str,
7105        why: String,
7106    ) -> Result<()> {
7107        // `Started` stays recorded: a resume retries the restore.
7108        rec.repair = crate::recovery::Repair::Started {
7109            before: head.to_owned(),
7110        };
7111        self.state.recovery = Some(rec);
7112        self.state
7113            .event("recovery", format!("repair could not be undone: {why}"));
7114        self.state.status = RunStatus::Blocked;
7115        self.state.save()?;
7116        bail!("recovery repair could not be rolled back: {why}")
7117    }
7118
7119    async fn review_loop(&mut self) -> Result<()> {
7120        if self.hold_base_sync().await? {
7121            return Ok(());
7122        }
7123        loop {
7124            if self
7125                .state
7126                .base_sync
7127                .as_ref()
7128                .is_some_and(|s| s.conflict.is_some())
7129            {
7130                return Ok(());
7131            }
7132            if self.catchup_repairable() {
7133                let record = self.state.reviews.last().expect("repairable round").clone();
7134                let winner = self
7135                    .state
7136                    .winner()
7137                    .cloned()
7138                    .context("catch-up repair has no winner")?;
7139                if !self.fix_review_round(&winner, record).await? {
7140                    return Ok(());
7141                }
7142            }
7143            self.review_loop_pass().await?;
7144            if !self.catchup_repairable() {
7145                return Ok(());
7146            }
7147        }
7148    }
7149
7150    /// Derive an owed catch-up repair from durable round evidence, including
7151    /// reentry after a crash before the fixer was dispatched.
7152    fn catchup_repairable(&self) -> bool {
7153        self.state.reviews.len() < self.state.config.graph.review_rounds
7154            && self.state.reviews.last().is_some_and(|r| {
7155                !r.clean
7156                    && r.blocking == 0
7157                    && r.fix.is_none()
7158                    && r.e2e_status() == E2eStatus::Failed
7159            })
7160    }
7161
7162    async fn review_loop_pass(&mut self) -> Result<()> {
7163        // A base that would not rebase is a person's decision, not a review
7164        // round: nothing here would change the answer, and reviewers and a
7165        // fixer would be spending real budget on a tree that cannot land
7166        // regardless of what they find.
7167        if self
7168            .state
7169            .base_sync
7170            .as_ref()
7171            .is_some_and(|s| s.conflict.is_some())
7172        {
7173            return Ok(());
7174        }
7175        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7176        // agent files with `magi task add` name the run that paid for it. The
7177        // prompt overlay is cloned alongside it because the waves borrow it
7178        // while `self` is mutably borrowed by the node's own bookkeeping.
7179        let run_id = self.state.id.clone();
7180        let prompts = self.state.config.prompts.clone();
7181        let Some(winner) = self.state.winner().cloned() else {
7182            return Ok(());
7183        };
7184        let granted_review = self
7185            .state
7186            .repair_grants
7187            .last()
7188            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7189            .cloned();
7190        let fresh_review = granted_review
7191            .as_ref()
7192            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7193        let max_rounds = granted_review
7194            .as_ref()
7195            .map_or(self.state.config.graph.review_rounds, |g| {
7196                g.review_start.saturating_add(1)
7197            });
7198        // A clean round, an exhausted round budget, or a stalled tree (see
7199        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7200        // they are recorded — recomputed here, not read off `status`, so a
7201        // reentry into a run that already stopped restates the identical
7202        // verdict instead of silently handing back whatever an earlier node
7203        // in this same walk clobbered `status` to (a solo-candidate
7204        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7205        // below runs an empty range once the budget is spent, and would
7206        // otherwise fall through without touching `status` at all.
7207        if let Some(status) =
7208            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7209        {
7210            // A reentry after a crash between the last round's save and
7211            // `stop_reviewing` reaches the hand-off here, not there.
7212            if status == RunStatus::Gating {
7213                self.record_contested_handoff();
7214            }
7215            self.state.status = status;
7216            self.state.save()?;
7217            return Ok(());
7218        }
7219        self.state.status = RunStatus::Reviewing;
7220        // A last recorded round whose own verification never resolved
7221        // (`ResourceBlocked` — the shared build cache, not the patch) is
7222        // never a concluded round, whatever the round budget says: starting
7223        // a fresh round on top of it would spend a whole new reviewer wave
7224        // re-reading an unchanged patch instead of just retrying the one
7225        // check that actually needs it, and once the budget is spent the
7226        // loop below has nothing left to do at all (its range is empty).
7227        // Retry that check directly instead, exactly the same retry
7228        // `stop_reviewing` already does for its own catch-up case.
7229        if !fresh_review
7230            && self.state.reviews.last().is_some_and(|r| {
7231                r.e2e_status() == E2eStatus::ResourceBlocked
7232                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7233            })
7234        {
7235            let shell = self.state.config.shell();
7236            return self
7237                .stop_unresolved_review(
7238                    "the last round's own verification never resolved",
7239                    &shell,
7240                    &winner.worktree,
7241                )
7242                .await;
7243        }
7244
7245        if granted_review.is_none() {
7246            self.recovery_repair(&winner).await?;
7247        }
7248        if self.resource_halt()? {
7249            return Ok(());
7250        }
7251
7252        let repo = self.state.repo.clone();
7253        let root = self.state.worktree_root();
7254        let language = self.state.config.graph.language.clone();
7255        let sessions = self.state.config.graph.sessions;
7256        let artifacts = self.merge_repair_artifacts();
7257        let base = self.landing_base();
7258        let base_short = short(&base);
7259        let reviewers = self.roles.reviewers.clone();
7260        let shell = self.state.config.shell();
7261
7262        for round in (self.state.reviews.len() + 1)..=max_rounds {
7263            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7264            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7265            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7266            // The prior round's own record, already persisted — never a
7267            // hand-carried variable of just its failing output: that is
7268            // exactly what let a round's e2e result drift out of sync with
7269            // which commit it was actually about (see `SCHEMA`'s doc for
7270            // schema 8). Judged against `head`, the commit reviewers are
7271            // about to look at now, so the summary always reads as "an
7272            // earlier head" here — this round's own patch has not been
7273            // checked yet.
7274            let prev_verification = self
7275                .state
7276                .reviews
7277                .last()
7278                .and_then(|r| r.verification_summary(&head));
7279            // Only the first round of a run that took a branch over: later
7280            // rounds have this run's own history.
7281            let prior_section = match (&self.state.recovery, round) {
7282                (Some(rec), 1) if self.state.reviews.is_empty() => {
7283                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7284                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7285                            Some(crate::recovery::head_of(
7286                                &git::diff(&winner.worktree, reviewed, "HEAD")
7287                                    .await
7288                                    .unwrap_or_default(),
7289                                crate::recovery::SINCE_DIFF_MAX,
7290                            ))
7291                        }
7292                        _ => None,
7293                    };
7294                    Some(prompt::recovery_review(rec, since.as_deref()))
7295                }
7296                _ => None,
7297            };
7298
7299            // Each reviewer gets its own detached checkout of exactly this
7300            // commit: nobody can perturb the winner's tree, and the fixer can
7301            // keep working without racing a reviewer.
7302            let mut jobs = Vec::new();
7303            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7304                self.coordinate(&repo).await?
7305            } else {
7306                None
7307            };
7308            for (r, spec) in reviewers.iter().cloned().enumerate() {
7309                let wt = root.join(format!("review-{}", r + 1));
7310                // `clean -fdx` in a reset wipes setup's products, so every
7311                // round sets the seat up again.
7312                if wt.exists() {
7313                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7314                } else {
7315                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7316                        None
7317                    } else {
7318                        self.coordinate(&repo).await?
7319                    };
7320                    git::worktree_add_detached(&repo, &wt, &head).await?;
7321                }
7322                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7323                    None
7324                } else {
7325                    self.coordinate(&repo).await?
7326                };
7327                self.setup_seat_worktree(
7328                    &repo,
7329                    &wt,
7330                    false,
7331                    coordination.as_ref().or(_setup_lease.as_ref()),
7332                )
7333                .await?;
7334                let seat_key = format!("review-{}", r + 1);
7335                // The seat starts the round on whoever answered it last, not
7336                // on the agent the spec names, so a failure is not re-paid.
7337                let spec = pick_start_spec(
7338                    &self.roles.reviewer_roster,
7339                    spec,
7340                    self.state.seat_history.get(&seat_key),
7341                );
7342                let seat = self.seat(&seat_key, &spec.id);
7343                jobs.push(SeatJob {
7344                    prompt: prompt::review(&prompt::ReviewCtx {
7345                        instruction: &self.state.instruction,
7346                        branch: &winner.branch,
7347                        base_short: &base_short,
7348                        stat: &stat,
7349                        patch: &patch,
7350                        verification: prev_verification.as_ref(),
7351                        prior: prior_section.as_deref(),
7352                        reviewers: reviewers.len(),
7353                        round,
7354                        rounds: max_rounds,
7355                        // A review-only run has no rankings, so nothing
7356                        // competed for this patch and the reviewer is told so.
7357                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7358                        lens: Lens::for_seat(r),
7359                        language: &language,
7360                        github_language: &self.state.config.graph.github_language,
7361                    }),
7362                    spec,
7363                    seat,
7364                    cwd: wt,
7365                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7366                    allow_write: false,
7367                    sessions,
7368                    artifacts: artifacts.clone(),
7369                    stem: format!("review-{round}-{}", r + 1),
7370                    handover: None,
7371                });
7372            }
7373
7374            drop(coordination);
7375            self.state.event(
7376                "review",
7377                format!(
7378                    "round {round}: {} reviewers on {}",
7379                    jobs.len(),
7380                    short(&head)
7381                ),
7382            );
7383            let mut quota_losses = Vec::new();
7384            let review_retries = self.state.config.graph.retries;
7385            let review_cache = self.state.config.cache_dir();
7386            let ctx = WaveCtx {
7387                carry_seats: true,
7388                pauses: Some((&self.pause, &self.interrupt)),
7389                run: &run_id,
7390                node: "review",
7391                prompts: &prompts,
7392                cache: review_cache.as_deref(),
7393                round: Some(round),
7394            };
7395            let results = ask_json_wave::<Review>(
7396                jobs,
7397                Arc::clone(&self.sem),
7398                review_retries,
7399                &self.roles.reviewer_roster,
7400                &ctx,
7401                &mut quota_losses,
7402                &mut self.state,
7403                &|_: &Review| Ok(()),
7404            )
7405            .await;
7406            if self.state.resource_block.is_some() {
7407                return self.stop_for_resource();
7408            }
7409            // Counted before the move below: how many of *this* round's
7410            // reviewer seats were lost to their own rate limit, as opposed to
7411            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7412            let round_quota_missing = quota_losses.len();
7413            self.state.quota.extend(quota_losses);
7414
7415            let mut records = Vec::new();
7416            let mut all_findings = Vec::new();
7417            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7418                let agent_id = seat.agent.clone();
7419                self.state.seats.insert(seat.key.clone(), seat);
7420                let mut record = ReviewRecord {
7421                    reviewer: r + 1,
7422                    agent: agent_id,
7423                    summary: String::new(),
7424                    findings: Vec::new(),
7425                    vote: None,
7426                    failed: None,
7427                    duration_ms: 0,
7428                    // Set for both outcomes: `failed: Some(_)` with
7429                    // `attempts > 0` is a seat every retry still lost, not a
7430                    // recovered one — only `failed: None` with `attempts > 0`
7431                    // reads as "answered after a nudge" (see this field's own
7432                    // doc).
7433                    attempts,
7434                };
7435                match res {
7436                    Ok((review, out)) => {
7437                        // Sanitized here, at the point every other piece of
7438                        // agent prose in this file is (candidate summaries,
7439                        // deliberation turns, vote reasons): a reviewer's own
7440                        // words are the one thing about it that could name
7441                        // it, and reconsideration below broadcasts this same
7442                        // summary and these same findings to every other
7443                        // seat on the panel.
7444                        record.summary =
7445                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7446                        record.vote = Some(review.vote);
7447                        record.duration_ms = out.duration_ms;
7448                        for (n, mut f) in review.findings.into_iter().enumerate() {
7449                            // ids are magi's, never the agent's: the fixer's
7450                            // adoption report is keyed by them.
7451                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7452                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7453                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7454                            // `file` is agent-supplied prose too, never
7455                            // checked against the real tree — the same
7456                            // exposure `title`/`detail` above have, just in
7457                            // a field easy to forget because it looks like a
7458                            // path rather than free text.
7459                            f.file = f
7460                                .file
7461                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7462                            all_findings.push(f.clone());
7463                            record.findings.push(f);
7464                        }
7465                        self.state.event(
7466                            "review",
7467                            format!(
7468                                "round {round}: reviewer {} voted {} with {} finding(s)",
7469                                r + 1,
7470                                review.vote.label(),
7471                                record.findings.len()
7472                            ),
7473                        );
7474                    }
7475                    Err(e) => {
7476                        record.failed = Some(e.to_string());
7477                        self.state.event(
7478                            "review",
7479                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7480                        );
7481                    }
7482                }
7483                records.push(record);
7484            }
7485
7486            // Tally the round's votes and, if they split, spend the one
7487            // round of reconsideration the split -> deliberate -> revote
7488            // shape `judge`/`vote` use for the panel, sized down to what a
7489            // read-only review round can afford: one round, and a revote
7490            // rather than an argument, because the panel already wrote its
7491            // reasoning down as findings the first time around.
7492            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7493            let vote_split =
7494                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7495            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7496            if vote_split {
7497                self.state.event(
7498                    "review",
7499                    format!(
7500                        "round {round}: votes split ({}) — one round of reconsideration",
7501                        initial_votes
7502                            .iter()
7503                            .map(|v| v.label())
7504                            .collect::<Vec<_>>()
7505                            .join(", ")
7506                    ),
7507                );
7508                // Seats read every seat's findings and votes, still numbered
7509                // and never named — the same anonymity `review` itself keeps.
7510                let panel: Vec<ReviewSeatReport<'_>> = records
7511                    .iter()
7512                    .filter_map(|r| {
7513                        r.vote.map(|vote| ReviewSeatReport {
7514                            reviewer: r.reviewer,
7515                            vote,
7516                            summary: &r.summary,
7517                            findings: &r.findings,
7518                        })
7519                    })
7520                    .collect();
7521
7522                let mut jobs = Vec::new();
7523                let mut seats_at = Vec::new();
7524                for (r, spec) in reviewers.iter().cloned().enumerate() {
7525                    // A seat with no initial vote has nothing to reconsider
7526                    // from and stays absent, the same as it stayed absent
7527                    // from `panel` above.
7528                    if records[r].vote.is_none() {
7529                        continue;
7530                    }
7531                    let wt = root.join(format!("review-{}", r + 1));
7532                    let seat_key = format!("review-{}", r + 1);
7533                    let spec = self.occupant(&seat_key, spec);
7534                    let seat = self.seat(&seat_key, &spec.id);
7535                    // A seat with no live session has already forgotten the
7536                    // initial review's prompt — restate the patch it is
7537                    // voting on, the same as `deliberate`/`vote` do for a
7538                    // judge in the same position.
7539                    // The panel already carries this seat's own review and
7540                    // vote, so restating the patch makes the prompt whole for
7541                    // a seat handed to another agent.
7542                    let build = |with_patch: bool| {
7543                        prompt::review_reconsider(&ReviewReconsiderCtx {
7544                            instruction: &self.state.instruction,
7545                            reviewer: r + 1,
7546                            lens: Lens::for_seat(r),
7547                            panel: &panel,
7548                            patch: with_patch.then_some(ReviewPatch {
7549                                branch: &winner.branch,
7550                                base_short: &base_short,
7551                                stat: &stat,
7552                                patch: &patch,
7553                            }),
7554                            round,
7555                            rounds: max_rounds,
7556                            language: &language,
7557                        })
7558                    };
7559                    let full = build(true);
7560                    let prompt = if has_context(&spec, &seat, sessions) {
7561                        build(false)
7562                    } else {
7563                        full.clone()
7564                    };
7565                    jobs.push(SeatJob {
7566                        prompt,
7567                        spec,
7568                        seat,
7569                        cwd: wt,
7570                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7571                        allow_write: false,
7572                        sessions,
7573                        artifacts: artifacts.clone(),
7574                        stem: format!("review-{round}-reconsider-{}", r + 1),
7575                        handover: Some(full),
7576                    });
7577                    seats_at.push(r);
7578                }
7579
7580                let mut recon_quota_losses = Vec::new();
7581                let recon_cache = self.state.config.cache_dir();
7582                let recon_ctx = WaveCtx {
7583                    carry_seats: true,
7584                    pauses: Some((&self.pause, &self.interrupt)),
7585                    run: &run_id,
7586                    node: "review",
7587                    prompts: &prompts,
7588                    cache: recon_cache.as_deref(),
7589                    round: Some(round),
7590                };
7591                let recon_results = ask_json_wave::<ReviewRevote>(
7592                    jobs,
7593                    Arc::clone(&self.sem),
7594                    review_retries,
7595                    &self.roles.reviewer_roster,
7596                    &recon_ctx,
7597                    &mut recon_quota_losses,
7598                    &mut self.state,
7599                    &|_: &ReviewRevote| Ok(()),
7600                )
7601                .await;
7602                if self.state.resource_block.is_some() {
7603                    return self.stop_for_resource();
7604                }
7605                self.state.quota.extend(recon_quota_losses);
7606
7607                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7608                    let agent_id = seat.agent.clone();
7609                    self.state.seats.insert(seat.key.clone(), seat);
7610                    let mut rec = ReviewRevoteRecord {
7611                        reviewer: r + 1,
7612                        agent: agent_id,
7613                        vote: None,
7614                        reason: String::new(),
7615                        failed: None,
7616                    };
7617                    match res {
7618                        Ok((rv, _)) => {
7619                            rec.vote = Some(rv.vote);
7620                            rec.reason =
7621                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7622                            self.state.event(
7623                                "review",
7624                                format!(
7625                                    "round {round}: reviewer {} revoted {}",
7626                                    r + 1,
7627                                    rv.vote.label()
7628                                ),
7629                            );
7630                        }
7631                        Err(e) => {
7632                            rec.failed = Some(e.to_string());
7633                            self.state.event(
7634                                "review",
7635                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7636                            );
7637                        }
7638                    }
7639                    reconsideration.push(rec);
7640                }
7641            } else if initial_votes.len() > 1 {
7642                self.state.event(
7643                    "review",
7644                    format!(
7645                        "round {round}: votes agreed ({}) — no reconsideration",
7646                        initial_votes[0].label()
7647                    ),
7648                );
7649            }
7650
7651            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7652            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7653            // A round that already has a blocking finding and a round left to
7654            // try is going back to the fixer no matter what `verify.e2e`
7655            // says, so running it first only spends the loop's slowest step
7656            // (minutes, for a Rust repo's full test suite) on a head about
7657            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7658            // runs once a round has no blocking findings left (see
7659            // `round_is_clean`, which a deferred — empty — `e2e` can never
7660            // satisfy since `blocking` is nonzero whenever this branch is
7661            // taken), and `stop_reviewing` forces a real run before it will
7662            // ever read a deferred round as green.
7663            let defer_e2e =
7664                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7665            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7666                let reason =
7667                    format!("{blocking} blocking finding(s) already required a fix this round");
7668                self.state.event(
7669                    "verify",
7670                    format!(
7671                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7672                         {}); it will run once a round has none left",
7673                        short(&head)
7674                    ),
7675                );
7676                (Vec::new(), false, true, Some(reason))
7677            } else {
7678                let e2e_commands = self.state.config.verify.e2e.clone();
7679                let cache_dir = self.state.config.cache_dir();
7680                let context = format!("round {round}");
7681                let wait = self.repository_wait();
7682                let (e2e, verify_retried) = with_cache_lease(
7683                    &mut self.state,
7684                    cache_dir.as_deref(),
7685                    "e2e",
7686                    "e2e",
7687                    &winner.worktree,
7688                    &head,
7689                    verify_timeout,
7690                    &context,
7691                    &wait,
7692                    |state, budget| {
7693                        let shell = shell.clone();
7694                        let e2e_commands = e2e_commands.clone();
7695                        let worktree = winner.worktree.clone();
7696                        let context = context.clone();
7697                        let winner = winner.clone();
7698                        async move {
7699                            if !e2e_commands.is_empty() {
7700                                // The suite certifies one SHA. Format before it
7701                                // runs, or the commit `pre_gate` makes at the
7702                                // gate owes the whole suite again.
7703                                Self::run_pre_gate(state, &winner).await;
7704                            }
7705                            run_e2e_with_retry(
7706                                state,
7707                                &shell,
7708                                &e2e_commands,
7709                                &worktree,
7710                                budget,
7711                                &context,
7712                                Some(round),
7713                            )
7714                            .await
7715                        }
7716                    },
7717                )
7718                .await?;
7719                (e2e, verify_retried, false, None)
7720            };
7721
7722            let expected = records.len();
7723            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7724            let incomplete = answered < expected;
7725            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7726            let policy = self.state.config.graph.incomplete_review;
7727            let clean = round_is_clean(
7728                blocking,
7729                e2e_ok,
7730                answered,
7731                expected,
7732                round_quota_missing,
7733                policy,
7734            );
7735
7736            let mut round_record = ReviewRound {
7737                round,
7738                head: head.clone(),
7739                verified_head: None,
7740                verified_at: None,
7741                reviews: records,
7742                e2e,
7743                verify_retried,
7744                e2e_deferred,
7745                e2e_defer_reason,
7746                fix: None,
7747                blocking,
7748                answered,
7749                expected,
7750                clean,
7751                progressed: false,
7752                vote_split,
7753                reconsideration,
7754                verdict: None,
7755            };
7756            // The final vote per seat is its revote where reconsideration
7757            // ran and answered, its initial vote otherwise — the same
7758            // fallback `tally` uses for a judge whose private vote failed.
7759            round_record.verdict = ReviewVote::worst(
7760                round_record
7761                    .final_votes()
7762                    .into_iter()
7763                    .map(|(_, _, vote)| vote),
7764            );
7765            // Which commit and when magi actually attempted to check —
7766            // known the moment a command was dispatched against `head`,
7767            // whether or not it finished: a resource-blocked attempt still
7768            // targeted a specific commit at a specific time, and leaving
7769            // that unrecorded is exactly what made `verification_summary`
7770            // report a fresh attempt as "commit unknown ... recorded before
7771            // this was tracked", indistinguishable from a genuinely old,
7772            // untracked record. Only a deferred or unconfigured round never
7773            // ran at all and has nothing to record — see
7774            // `ReviewRound::verified_head`'s own doc.
7775            if !matches!(
7776                round_record.e2e_status(),
7777                E2eStatus::Deferred | E2eStatus::NotConfigured
7778            ) {
7779                // HEAD, not the reviewed `head`: `pre_gate` may have committed
7780                // mechanical fixes before the commands ran.
7781                round_record.verified_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
7782                round_record.verified_at = Some(Timestamp::now());
7783            }
7784
7785            if incomplete {
7786                let missing: Vec<String> = round_record
7787                    .reviews
7788                    .iter()
7789                    .filter(|r| r.failed.is_some())
7790                    .map(|r| format!("review-{}", r.reviewer))
7791                    .collect();
7792                self.state.event(
7793                    "review",
7794                    format!(
7795                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7796                        missing.join(", ")
7797                    ),
7798                );
7799            }
7800
7801            if clean {
7802                self.state.event(
7803                    "review",
7804                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7805                        format!(
7806                            "round {round}: clean (warn policy, incomplete panel) — no \
7807                             blocking findings from the seats that answered, verification green"
7808                        )
7809                    } else if incomplete {
7810                        format!(
7811                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7812                             quorum) — no blocking findings from the seats that answered, \
7813                             verification green",
7814                            expected - answered
7815                        )
7816                    } else {
7817                        format!("round {round}: clean — no blocking findings, verification green")
7818                    },
7819                );
7820                self.state.reviews.push(round_record);
7821                self.state.status = RunStatus::Gating;
7822                self.state.save()?;
7823                return Ok(());
7824            }
7825
7826            // Nothing was raised and verification passed, but not every seat
7827            // answered and `round_is_clean` still refused to call it clean —
7828            // either a seat is missing for a reason other than its own quota
7829            // (a crash, a timeout, unparsable output — worth another try), or
7830            // every seat that could have answered lost its quota and nobody
7831            // is left to decide on: re-review rather than send the fixer
7832            // after a round with nothing to fix.
7833            if incomplete && blocking == 0 && e2e_ok {
7834                self.state.reviews.push(round_record);
7835                self.state.save()?;
7836                if round == max_rounds {
7837                    self.state.status = RunStatus::Blocked;
7838                    self.state.event(
7839                        "review",
7840                        format!(
7841                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7842                             refusing to call it clean",
7843                            expected - answered
7844                        ),
7845                    );
7846                    return Ok(());
7847                }
7848                continue;
7849            }
7850
7851            // Nothing for the fixer to act on (`blocking == 0`) and the only
7852            // reason this round is not clean is that magi itself never got
7853            // a command to run — the shared build cache, not the patch (see
7854            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7855            // the fixer would invite a change to appease contention that has
7856            // nothing to do with the diff, and would leave this attempt
7857            // sitting in the next round's prompt as if it were about an
7858            // earlier, superseded commit rather than what it actually is:
7859            // the same head, still waiting to be checked. Wait for it the
7860            // same way the final round's own contention is already handled,
7861            // whatever round this happens to be.
7862            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7863                self.state.reviews.push(round_record);
7864                return self
7865                    .stop_unresolved_review(
7866                        "the round's own verification could not run",
7867                        &shell,
7868                        &winner.worktree,
7869                    )
7870                    .await;
7871            }
7872
7873            if round == max_rounds {
7874                self.state.reviews.push(round_record);
7875                return self
7876                    .stop_reviewing(
7877                        &format!(
7878                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7879                        ),
7880                        &shell,
7881                        &winner.worktree,
7882                    )
7883                    .await;
7884            }
7885
7886            if !self.fix_review_round(&winner, round_record).await? {
7887                return Ok(());
7888            }
7889        }
7890        Ok(())
7891    }
7892
7893    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7894    /// False means admission or continuation policy stopped the review loop.
7895    async fn fix_review_round(
7896        &mut self,
7897        winner: &Candidate,
7898        mut round_record: ReviewRound,
7899    ) -> Result<bool> {
7900        let round = round_record.round;
7901        let shell = self.state.config.shell();
7902        let step = self.run_fix_round(winner, &mut round_record).await?;
7903        if matches!(step, FixStep::Resource) {
7904            self.stop_for_resource()?;
7905            return Ok(false);
7906        }
7907        if self.state.reviews.len() < round {
7908            self.state.reviews.push(round_record);
7909        } else {
7910            self.state.reviews[round - 1] = round_record;
7911        }
7912        self.state.save()?;
7913
7914        // The fixer's own report never came back this round, even after
7915        // `continue_fix_report`'s own budget was spent on it — not an
7916        // ordinary "no report" (dropped stream, quota, plain failure),
7917        // which already reads that way and is left to the existing round
7918        // budget. Stopping here, rather than opening another round, is
7919        // what keeps a next reviewer/fixer wave from ever being
7920        // dispatched onto `winner.worktree` while whatever the seat's
7921        // last call may still have running there is unaccounted for: no
7922        // process liveness check exists (and none is being added — see
7923        // AGENTS.md/this task's own scope), so the only way to honour
7924        // "nothing starts before a valid report returns" is to not start
7925        // anything further on this worktree from this run at all.
7926        if matches!(step, FixStep::Recorded { report_lost: true }) {
7927            self.stop_reviewing(
7928                "the fixer's adoption report never came back, even after resuming its \
7929                         own seat; refusing to start another round against the same worktree \
7930                         while that is unresolved",
7931                &shell,
7932                &winner.worktree,
7933            )
7934            .await?;
7935            return Ok(false);
7936        }
7937
7938        let streak = self
7939            .state
7940            .reviews
7941            .iter()
7942            .rev()
7943            .take_while(|r| !r.progressed)
7944            .count();
7945        if streak >= STAGNANT_LIMIT {
7946            self.stop_reviewing(
7947                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7948                &shell,
7949                &winner.worktree,
7950            )
7951            .await?;
7952            return Ok(false);
7953        }
7954        Ok(true)
7955    }
7956
7957    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7958    async fn run_fix_round(
7959        &mut self,
7960        winner: &Candidate,
7961        round_record: &mut ReviewRound,
7962    ) -> Result<FixStep> {
7963        let round = round_record.round;
7964        let max_rounds = self.state.config.graph.review_rounds;
7965        let run_id = self.state.id.clone();
7966        let prompts = self.state.config.prompts.clone();
7967        let language = self.state.config.graph.language.clone();
7968        let sessions = self.state.config.graph.sessions;
7969        let artifacts = self.merge_repair_artifacts();
7970        let base = self.landing_base();
7971        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7972        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7973        let this_round_verification = round_record.verification_summary(&head);
7974        let all_findings: Vec<_> = round_record
7975            .reviews
7976            .iter()
7977            .flat_map(|r| r.findings.iter().cloned())
7978            .collect();
7979        // Fix. The winner's own implementer seat continues its conversation:
7980        // the competition is over, so context is pure benefit now.
7981        let blocking_findings: Vec<_> = all_findings
7982            .iter()
7983            .filter(|f| f.severity.blocks())
7984            .cloned()
7985            .collect();
7986        let fix_prompt = prompt::fix_in(
7987            &self.state.instruction,
7988            &blocking_findings,
7989            this_round_verification.as_ref(),
7990            round,
7991            max_rounds,
7992            &language,
7993            &self.state.config.graph.github_language,
7994        );
7995        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7996        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7997        let (job, seat, out) = self
7998            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7999                prompt: fix_prompt.clone(),
8000                spec,
8001                seat,
8002                cwd: winner.worktree.clone(),
8003                timeout,
8004                allow_write: true,
8005                sessions,
8006                artifacts: artifacts.clone(),
8007                stem: format!("fix-{round}"),
8008                handover: None,
8009            })
8010            .await;
8011        // Not asked: the round is not recorded (so a resume repeats it),
8012        // no fixer was handed over to, and the seat is left as it was.
8013        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8014            return Ok(FixStep::Resource);
8015        }
8016        let agent_id = seat.agent.clone();
8017
8018        let mut fix = FixRecord {
8019            agent: agent_id,
8020            addressed: Vec::new(),
8021            rejected: Vec::new(),
8022            notes: String::new(),
8023            committed: false,
8024            failed: None,
8025            duration_ms: 0,
8026            continuation: None,
8027        };
8028        let mut continuation = ContinuationRecord::not_needed();
8029        let mut final_seat = seat.clone();
8030        match out {
8031            AgentOutcome::Ok(o) => {
8032                fix.duration_ms = o.duration_ms;
8033                let parsed = verdict::extract_json::<FixReport>(&o.text);
8034                // A parsed report standing next to a command this same
8035                // reply's own CLI never confirmed the exit status of is
8036                // not a resolved answer — the identical `CommandEvidence`
8037                // `state.jobs` renders, read here instead of only on
8038                // display, per the completion judgment and the shown
8039                // record needing to agree.
8040                let incomplete_reason = match &parsed {
8041                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
8042                        "the reply parsed, but it reported a command whose own CLI never \
8043                             confirmed an exit status"
8044                            .to_owned(),
8045                    ),
8046                    Ok(_) => None,
8047                    Err(e) => Some(e.to_string()),
8048                };
8049                match incomplete_reason {
8050                    None => {
8051                        let report = parsed.expect("checked Ok above");
8052                        fix.addressed = report.addressed;
8053                        fix.rejected = report.rejected;
8054                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8055                    }
8056                    Some(reason) => {
8057                        let (resumed_seat, resolved, failure, cont) = self
8058                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
8059                            .await;
8060                        fix.duration_ms += cont.cumulative_wait_ms;
8061                        continuation = cont;
8062                        final_seat = resumed_seat;
8063                        match resolved {
8064                            Some(report) => {
8065                                fix.addressed = report.addressed;
8066                                fix.rejected = report.rejected;
8067                                fix.notes =
8068                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
8069                            }
8070                            None => fix.failed = failure,
8071                        }
8072                    }
8073                }
8074            }
8075            // The CLI's raw error JSON is not a fix report to parse.
8076            AgentOutcome::Dropped(o) => {
8077                fix.duration_ms = o.duration_ms;
8078                let why = o
8079                    .dropped
8080                    .as_ref()
8081                    .map(|d| d.why.as_str())
8082                    .unwrap_or("the CLI ended the stream without delivering its answer");
8083                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
8084            }
8085            AgentOutcome::Quota(o) => {
8086                self.state.quota.push(QuotaLoss::from_report(
8087                    final_seat.key.clone(),
8088                    "fix",
8089                    o.quota.as_ref(),
8090                ));
8091                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
8092            }
8093            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
8094        }
8095        if self.state.resource_block.is_some() {
8096            self.state.seats.insert(final_seat.key.clone(), final_seat);
8097            return Ok(FixStep::Resource);
8098        }
8099        fix.continuation = Some(continuation);
8100        self.state.seats.insert(final_seat.key.clone(), final_seat);
8101        let coordination = self.coordinate(&winner.worktree).await?;
8102        if let Ok(r) = git::rescue_commit(
8103            &winner.worktree,
8104            &format!("magi: review round {round} fixes (uncommitted work)"),
8105        )
8106        .await
8107        {
8108            self.state.note_withheld("fix", &r.withheld);
8109        }
8110        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8111        fix.committed = after != before;
8112        // Judged by what `git` says moved against base, never by the
8113        // fixer's own `addressed`/`rejected` count — see
8114        // `ReviewRound::progressed`. Propagated with `?`, the same as the
8115        // `patch` snapshot above: swallowing this error would default
8116        // `diff_after` to empty, which almost always differs from a
8117        // non-empty `patch` and reads as "progressed" — exactly backwards
8118        // for a `git` failure the stagnation check cannot see through.
8119        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
8120        drop(coordination);
8121        let progressed = diff_after != patch;
8122        let commit_note = if fix.committed {
8123            "committed"
8124        } else {
8125            "NO new commit"
8126        };
8127        let tree_note = if progressed {
8128            "changed vs base"
8129        } else {
8130            "unchanged vs base"
8131        };
8132        self.state.event(
8133            "fix",
8134            match &fix.failed {
8135                // Distinct on purpose from "0 addressed, 0 rejected": the
8136                // fixer's own diff still landed (blocking counts do keep
8137                // falling round over round), only its adoption report did
8138                // not come back, so this must never read like every
8139                // finding was reviewed and declined.
8140                Some(reason) => {
8141                    format!(
8142                        "round {round}: fixer's adoption report was lost ({reason}); \
8143                             {commit_note}, tree {tree_note}"
8144                    )
8145                }
8146                None => format!(
8147                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
8148                         {tree_note}{}",
8149                    fix.addressed.len(),
8150                    fix.rejected.len(),
8151                    if continuation.outcome == ContinuationOutcome::Resumed {
8152                        format!(
8153                            " (adoption report recovered after {} continuation(s))",
8154                            continuation.attempts
8155                        )
8156                    } else {
8157                        String::new()
8158                    },
8159                ),
8160            },
8161        );
8162        round_record.fix = Some(fix);
8163        round_record.progressed = progressed;
8164        Ok(FixStep::Recorded {
8165            report_lost: matches!(
8166                continuation.outcome,
8167                ContinuationOutcome::Exhausted
8168                    | ContinuationOutcome::QuotaLost
8169                    | ContinuationOutcome::NoSession
8170            ),
8171        })
8172    }
8173
8174    /// Decide, from the last recorded round's own verification, whether
8175    /// stopping the review loop is a hand-off or a genuine block.
8176    ///
8177    /// Called once the loop has given up trying — the round budget is spent,
8178    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8179    /// findings still open, never while a round is still clean or the
8180    /// incomplete-panel case handled inline above. Gate and e2e are facts
8181    /// about the tree; a lingering review finding is an opinion, and this
8182    /// workload's own `magi stats` puts reviewer precision low enough
8183    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8184    /// must not by itself stand between a green, verified change and the
8185    /// human who decides what to do with it. A red e2e is not an opinion, so
8186    /// that case still blocks, with the failing command and a tail of its
8187    /// output recorded here rather than left in `run.json` for someone to go
8188    /// find.
8189    ///
8190    /// A round that deferred its own e2e (see [`Config::graph`]'s
8191    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8192    /// only because nothing ran, and treating an empty list as a passing one
8193    /// here is exactly the "deferred painted green" bug this function exists
8194    /// to not have. When the last round's own verification never resolved —
8195    /// deferred on purpose, or a real attempt the shared build cache blocked
8196    /// — this makes (or retries) the real run, on the actual worktree this
8197    /// loop is about to stop touching, before deciding anything. A
8198    /// resource-blocked attempt is likewise never read as either green or
8199    /// red: it is evidence about the machine, not the patch (see
8200    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8201    /// blocked cache leaves this call without deciding rather than guessing
8202    /// — the caller retries on a later reentry.
8203    /// Record, once, that the review loop handed off over a blocking finding
8204    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8205    /// `land` asks the owner even with `land_approval` off. Called from every
8206    /// path that concludes `Gating`; a reentry keeps the first record.
8207    fn record_contested_handoff(&mut self) {
8208        if self.state.contested_handoff.is_some() {
8209            return;
8210        }
8211        let Some(contested) = self
8212            .state
8213            .reviews
8214            .last()
8215            .and_then(ReviewRound::contested_handoff)
8216        else {
8217            return;
8218        };
8219        self.state.event(
8220            "review",
8221            format!(
8222                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8223                 wait for the owner's approval",
8224                contested.findings.len(),
8225                contested.rejecters.len()
8226            ),
8227        );
8228        self.state.contested_handoff = Some(contested);
8229    }
8230
8231    async fn stop_unresolved_review(
8232        &mut self,
8233        why: &str,
8234        shell: &[String],
8235        worktree: &Path,
8236    ) -> Result<()> {
8237        self.stop_reviewing_inner(why, shell, worktree, true).await
8238    }
8239
8240    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8241        self.stop_reviewing_inner(why, shell, worktree, false).await
8242    }
8243
8244    async fn stop_reviewing_inner(
8245        &mut self,
8246        why: &str,
8247        shell: &[String],
8248        worktree: &Path,
8249        repairable: bool,
8250    ) -> Result<()> {
8251        let round_idx = self.state.reviews.len() - 1;
8252        // A deferred round and a resource-blocked one are the same shape
8253        // here: neither has a real result yet, and both get one more
8254        // attempt. Read off `e2e_status` — the single source for this —
8255        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8256        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8257        // still retries instead of being read as a settled result the
8258        // instant it stops being empty.
8259        let needs_catchup_run = matches!(
8260            self.state.reviews[round_idx].e2e_status(),
8261            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8262        );
8263        if needs_catchup_run {
8264            let round = self.state.reviews[round_idx].round;
8265            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8266            let commands = self.state.config.verify.e2e.clone();
8267            let before_head = git::rev_parse(worktree, "HEAD").await?;
8268            let winner = self.state.winner().cloned();
8269            let cache_dir = self.state.config.cache_dir();
8270            let context = format!(
8271                "round {round}: verification unresolved, catching up before the final decision"
8272            );
8273            let wait = self.repository_wait();
8274            let (outcomes, verify_retried) = with_cache_lease(
8275                &mut self.state,
8276                cache_dir.as_deref(),
8277                "e2e",
8278                "e2e",
8279                worktree,
8280                &before_head,
8281                timeout,
8282                &context,
8283                &wait,
8284                |state, budget| {
8285                    let shell = shell.to_vec();
8286                    let commands = commands.clone();
8287                    let context = context.clone();
8288                    async move {
8289                        if let Some(winner) = winner.filter(|_| !commands.is_empty()) {
8290                            // Same reason as in the round itself: format before
8291                            // the suite, not after it.
8292                            Self::run_pre_gate(state, &winner).await;
8293                        }
8294                        run_e2e_with_retry(
8295                            state,
8296                            &shell,
8297                            &commands,
8298                            worktree,
8299                            budget,
8300                            &context,
8301                            Some(round),
8302                        )
8303                        .await
8304                    }
8305                },
8306            )
8307            .await?;
8308            // After `pre_gate`: the commit the commands actually ran against.
8309            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8310            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8311                self.state.record_base_progress(&attempted_head);
8312            }
8313            let last = &mut self.state.reviews[round_idx];
8314            last.e2e = outcomes;
8315            last.verify_retried = verify_retried;
8316            // Always the commit and time this attempt actually targeted,
8317            // whether or not it happens to equal the reviewed `head` and
8318            // whether or not a command finished — see
8319            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8320            // attempt is recorded too, so a later reader sees "attempted
8321            // again at T2" rather than silence.
8322            last.verified_head = Some(attempted_head);
8323            last.verified_at = Some(Timestamp::now());
8324            if verify_inconclusive(&last.e2e) {
8325                // Still not a real result: `e2e_deferred` is left exactly
8326                // as it was, so `needs_catchup_run` above reads
8327                // `ResourceBlocked` (via `e2e_status`, which checks
8328                // `resource_blocked` before `e2e_deferred`) and retries
8329                // again on the next reentry, rather than recording
8330                // contention as a red e2e and blocking the run on it.
8331                self.state.save()?;
8332                return Ok(());
8333            }
8334            last.e2e_deferred = false;
8335            // A complete panel may have been saved as unclean solely because
8336            // verification could not run. Refresh that verdict with the new
8337            // outcomes before a granted repair consumes the round's result.
8338            // Incomplete panels retain their original policy/quota decision.
8339            if last.answered == last.expected {
8340                last.clean = round_is_clean(
8341                    last.blocking,
8342                    last.e2e.iter().all(CommandOutcome::ok),
8343                    last.answered,
8344                    last.expected,
8345                    0,
8346                    self.state.config.graph.incomplete_review,
8347                );
8348            }
8349        }
8350        let last = &self.state.reviews[round_idx];
8351        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8352
8353        match last.e2e_status() {
8354            E2eStatus::Failed => {
8355                let red: Vec<String> = last
8356                    .e2e
8357                    .iter()
8358                    .filter(|o| !o.ok())
8359                    .map(|o| {
8360                        format!(
8361                            "`{}` -> {:?}\n{}",
8362                            o.command,
8363                            o.code,
8364                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8365                        )
8366                    })
8367                    .collect();
8368                let can_repair = last.blocking == 0 && last.fix.is_none();
8369                self.state.reviews[round_idx].clean = false;
8370                self.state
8371                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8372                self.state.status = RunStatus::Blocked;
8373                if repairable
8374                    && can_repair
8375                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8376                {
8377                    self.state.status = RunStatus::Reviewing;
8378                }
8379            }
8380            // `needs_catchup_run` above already retried once this call; if
8381            // it is still blocked, this is magi's own admission it could
8382            // not get a command to run, never a verdict on the patch — the
8383            // run is left exactly where a later reentry can retry again.
8384            E2eStatus::ResourceBlocked => {
8385                self.state.event(
8386                    "review",
8387                    format!(
8388                        "{why}; e2e could not run (resource unavailable); not \
8389                         deciding yet"
8390                    ),
8391                );
8392            }
8393            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8394                self.state.event(
8395                    "review",
8396                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8397                );
8398                self.record_contested_handoff();
8399                self.state.status = RunStatus::Gating;
8400            }
8401        }
8402        self.state.save()?;
8403        Ok(())
8404    }
8405
8406    // ----------------------------------------------------------------- gate
8407
8408    async fn gate(&mut self) -> Result<()> {
8409        if self.hold_base_sync().await? {
8410            return Ok(());
8411        }
8412        // Judged by the review record itself, not by `status`: a solo
8413        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8414        // reentry (see `judge`), and trusting it here is exactly how a run
8415        // that exhausted its review budget got gated and merged a second
8416        // time around. `review_conclusion` recomputes the review loop's own
8417        // verdict from the round records themselves — `Gating` for a clean
8418        // round or a hand-off (see `stop_reviewing`), anything else means the
8419        // loop is still going or genuinely blocked.
8420        // A base the winner could not be replayed onto is a decision, not a
8421        // round: there is no landing tree to gate. Read as its own record for
8422        // the same reason the review verdict is.
8423        if self.state.status == RunStatus::Failed
8424            || self
8425                .state
8426                .base_sync
8427                .as_ref()
8428                .is_some_and(|s| s.conflict.is_some())
8429            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8430                != Some(RunStatus::Gating)
8431        {
8432            return Ok(());
8433        }
8434        let Some(winner) = self.state.winner().cloned() else {
8435            return Ok(());
8436        };
8437        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8438        self.state.reconcile_verification_head(&head);
8439        self.state.save()?;
8440        let wait = self.repository_wait();
8441        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8442            return Ok(());
8443        }
8444        if self.state.gate_ran {
8445            // `review_loop` derives its conclusion from the clean review
8446            // record on every reentry and therefore puts a completed run back
8447            // in `Gating`. A recorded gate is a stronger, terminal fact:
8448            // retain its original command output (or lack of any, for a repo
8449            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8450            // doc) and restore `Blocked` on a real failure rather than
8451            // pretending the command is still running or running it a second
8452            // time. `gate_ran == false` remains the only shape — unattempted,
8453            // or a resource-blocked retry — that may still need to execute a
8454            // command.
8455            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8456                self.state.status = RunStatus::Blocked;
8457                self.state.save()?;
8458            }
8459            return Ok(());
8460        }
8461        self.state.status = RunStatus::Gating;
8462        let mut outcomes = self.run_gate(&winner).await?;
8463        let mut attempted_head = self.state.gate_head.clone();
8464        loop {
8465            // A resource-blocked outcome means the gate command never actually
8466            // ran - the shared build cache could not be acquired or confirmed
8467            // fresh in time - which is evidence about the machine, not about
8468            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8469            // Recording it as a red gate would mark a run `Blocked` on nothing
8470            // but contention magi has already logged; leaving `self.state.gate`
8471            // empty and `self.state.gate_ran` false instead keeps the shape
8472            // this function already treats as "still needs to run" (see the
8473            // early-return above), so the next call retries the command
8474            // rather than concluding anything.
8475            if verify_inconclusive(&outcomes) {
8476                self.state.save()?;
8477                return Ok(());
8478            }
8479            if outcomes.iter().all(CommandOutcome::ok) {
8480                break;
8481            }
8482            let fixes_before = self.state.gate_fixes.len();
8483            match self.gate_fix_round(&winner, &outcomes).await? {
8484                GateFix::Retry => {
8485                    outcomes = self.run_gate(&winner).await?;
8486                    attempted_head = self.state.gate_head.clone();
8487                }
8488                GateFix::Stop => {
8489                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8490                    if self.state.gate_fixes.len() > fixes_before
8491                        && attempted_head.as_deref() != Some(current.as_str())
8492                        && verify_current_head(&mut self.state, &winner, &wait).await?
8493                    {
8494                        // A new SHA owes a gate even when a spent fixer changed only
8495                        // commit metadata. This does not grant another repair round.
8496                        outcomes = self.run_gate(&winner).await?;
8497                        attempted_head = self.state.gate_head.clone();
8498                    } else {
8499                        break;
8500                    }
8501                }
8502                GateFix::Defer => {
8503                    self.state.save()?;
8504                    return Ok(());
8505                }
8506            }
8507        }
8508        let passed = outcomes.iter().all(CommandOutcome::ok);
8509        self.state.gate = outcomes;
8510        self.state.gate_ran = true;
8511        self.state.gate_head = attempted_head;
8512        if !self.state.gate.is_empty()
8513            && !verify_inconclusive(&self.state.gate)
8514            && let Some(head) = self.state.gate_head.clone()
8515        {
8516            self.state.record_base_progress(&head);
8517        }
8518        self.state.save()?;
8519        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8520            return Ok(());
8521        }
8522        if !passed {
8523            self.state.status = RunStatus::Blocked;
8524            let spent = self.state.gate_fixes.len();
8525            self.state.event(
8526                "gate",
8527                if spent == 0 {
8528                    "gate failed; not merging".to_owned()
8529                } else {
8530                    format!("gate failed after {spent} gate-fix round(s); not merging")
8531                },
8532            );
8533        }
8534        self.state.save()?;
8535        Ok(())
8536    }
8537
8538    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8539    /// changed into one commit. Runs before `verify.e2e` in the review loop
8540    /// and again before the gate commands, always on the winner and never on
8541    /// a candidate awaiting judging.
8542    ///
8543    /// Before e2e as well as before the gate because command evidence belongs
8544    /// to a SHA ([`verify_current_head`]): a commit made only at the gate is a
8545    /// commit e2e never saw, and the suite ran again for a formatter's diff -
8546    /// 5 of the 22 gated runs on this repository in November 2026. The gate's
8547    /// own call then finds nothing left to change. The commands must be
8548    /// idempotent; one that is not makes a commit per call.
8549    ///
8550    /// Never fails the run: a non-zero exit or timeout is a warning and a
8551    /// recorded outcome, and the gate remains the single arbiter. Nothing
8552    /// configured means nothing happens - no event, no commit. `commit_all`
8553    /// commits any leftover change under the neutral identity and returns
8554    /// `false` when the tree is clean, so no empty commit is ever made.
8555    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8556        let commands = state.config.verify.pre_gate.clone();
8557        if commands.is_empty() {
8558            return;
8559        }
8560        let shell = state.config.shell();
8561        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8562        let (outcomes, _) = run_commands(
8563            state,
8564            "pre_gate",
8565            "pre_gate",
8566            0,
8567            &shell,
8568            &commands,
8569            &winner.worktree,
8570            timeout,
8571        )
8572        .await;
8573        for o in &outcomes {
8574            if !o.ok() {
8575                tracing::warn!(
8576                    "pre_gate `{}` failed ({:?}); the gate decides",
8577                    o.command,
8578                    o.code
8579                );
8580            }
8581            state.event(
8582                "pre_gate",
8583                format!(
8584                    "`{}` -> {}",
8585                    o.command,
8586                    if o.ok() {
8587                        "pass".to_owned()
8588                    } else {
8589                        format!(
8590                            "FAIL ({:?})\n{}",
8591                            o.code,
8592                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8593                        )
8594                    }
8595                ),
8596            );
8597        }
8598        state.pre_gate = outcomes;
8599        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8600            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8601                Ok(head) => {
8602                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8603                    state.pre_gate_commit = Some(head);
8604                }
8605                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8606            },
8607            Ok(false) => {}
8608            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8609        }
8610        if let Err(e) = state.save() {
8611            tracing::warn!("could not persist the pre_gate record: {e:#}");
8612        }
8613    }
8614
8615    /// Run `verify.gate` once against the winner's current tree, logging one
8616    /// event per command. Empty when nothing is configured.
8617    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8618        let wait = self.repository_wait();
8619        run_landing_gate(&mut self.state, winner, &wait).await
8620    }
8621
8622    /// One bounded fix round for a failing gate.
8623    ///
8624    /// The fixer is told the failure came from the gate itself, not from a
8625    /// reviewer, and is shown the failed commands, their exit codes and a tail
8626    /// of their output - whatever `[verify].gate` holds, nothing here knows
8627    /// what those commands run. Only a normal non-zero exit that printed
8628    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8629    /// command or a full disk says nothing about the code, and a fixer sent
8630    /// after it can only appease the machine. The round is judged by what git
8631    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8632    /// again before the gate does, so a fix cannot trade a green gate for a
8633    /// red e2e unnoticed.
8634    async fn gate_fix_round(
8635        &mut self,
8636        winner: &Candidate,
8637        outcomes: &[CommandOutcome],
8638    ) -> Result<GateFix> {
8639        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8640            g.phase == crate::repair_budget::Phase::Fixing
8641                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8642        }) {
8643            self.state.gate_fixes.len().saturating_add(1)
8644        } else {
8645            self.state.config.graph.gate_fix_rounds
8646        };
8647        let spent = self.state.gate_fixes.len();
8648        if spent >= cap {
8649            if cap > 0 {
8650                self.state.event(
8651                    "gate",
8652                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8653                );
8654            }
8655            return Ok(GateFix::Stop);
8656        }
8657        if !gate_fixable(outcomes) {
8658            self.state.event(
8659                "gate",
8660                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8661                 command or similar); not spending a fix round on it",
8662            );
8663            return Ok(GateFix::Stop);
8664        }
8665        let min_free = self.state.config.disk.min_free_bytes;
8666        if min_free > 0 {
8667            match crate::disk::free_bytes(&winner.worktree) {
8668                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8669                Ok(free) => {
8670                    self.state.event(
8671                        "gate",
8672                        format!(
8673                            "only {free} bytes free ({min_free} required by `[disk] \
8674                             min_free_bytes`); not spending a fix round on a failure the disk \
8675                             may explain"
8676                        ),
8677                    );
8678                    return Ok(GateFix::Stop);
8679                }
8680                Err(e) => {
8681                    self.state.event(
8682                        "gate",
8683                        format!("free disk space could not be measured ({e:#}); no fix round"),
8684                    );
8685                    return Ok(GateFix::Stop);
8686                }
8687            }
8688        }
8689
8690        let attempt = spent + 1;
8691        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8692        let base = self.landing_base();
8693        let fix_prompt = prompt::gate_fix_in(
8694            &self.state.instruction,
8695            &failed,
8696            attempt,
8697            cap,
8698            &self.state.config.graph.language,
8699            &self.state.config.graph.github_language,
8700        );
8701        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8702        let sessions = self.state.config.graph.sessions;
8703        let artifacts = self.merge_repair_artifacts();
8704        self.state.event(
8705            "gate",
8706            format!("gate failed; gate-fix round {attempt} of {cap}"),
8707        );
8708        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8709        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8710        // Persist before invoking the fixer: it may commit and then be
8711        // interrupted before its result (or the following e2e) is saved.
8712        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8713            head: before.clone(),
8714            passed: false,
8715            outcomes: Vec::new(),
8716        });
8717        self.state.save()?;
8718        let (_, seat, out) = self
8719            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8720                prompt: fix_prompt.clone(),
8721                spec,
8722                seat,
8723                cwd: winner.worktree.clone(),
8724                timeout,
8725                allow_write: true,
8726                sessions,
8727                artifacts: artifacts.clone(),
8728                stem: format!("gate-fix-{attempt}"),
8729                handover: None,
8730            })
8731            .await;
8732        // The fixer was never asked: no round is spent (nothing is recorded)
8733        // and a later reentry asks the same agent again.
8734        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8735            return Ok(GateFix::Defer);
8736        }
8737        let mut record = GateFixRecord {
8738            agent: seat.agent.clone(),
8739            failed,
8740            notes: String::new(),
8741            committed: false,
8742            error: None,
8743        };
8744        match out {
8745            AgentOutcome::Ok(o) => {
8746                // A missing report is not a failed fix: the round is judged
8747                // by the tree below, and the report only carries prose.
8748                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8749                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8750                }
8751            }
8752            AgentOutcome::Dropped(_) => {
8753                record.error = Some("the CLI dropped the stream".to_owned());
8754            }
8755            AgentOutcome::Quota(o) => {
8756                self.state.quota.push(QuotaLoss::from_report(
8757                    seat.key.clone(),
8758                    "gate-fix",
8759                    o.quota.as_ref(),
8760                ));
8761                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8762            }
8763            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8764        }
8765        self.state.seats.insert(seat.key.clone(), seat);
8766        let coordination = self.coordinate(&winner.worktree).await?;
8767        if let Ok(r) = git::rescue_commit(
8768            &winner.worktree,
8769            &format!("magi: gate fix {attempt} (uncommitted work)"),
8770        )
8771        .await
8772        {
8773            self.state.note_withheld("gate-fix", &r.withheld);
8774        }
8775        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8776        record.committed = after != before;
8777        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8778        let note = record.error.clone();
8779        self.state.gate_fixes.push(record);
8780        self.state.save()?;
8781        if !changed {
8782            self.state.event(
8783                "gate",
8784                match note {
8785                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8786                    None => format!("gate-fix round {attempt}: the tree did not change"),
8787                },
8788            );
8789            return Ok(GateFix::Stop);
8790        }
8791        self.state.event(
8792            "gate",
8793            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8794        );
8795
8796        drop(coordination);
8797        let wait = self.repository_wait();
8798        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8799    }
8800
8801    // ---------------------------------------------------------------- merge
8802
8803    /// One read-only rewrite by the summary's author; a text the gate still
8804    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8805    /// applies on their word or on silence. `None` means the run is parked on
8806    /// that question and nothing may be pushed or opened yet.
8807    async fn guarded_pr_message(
8808        &mut self,
8809        winner: &Candidate,
8810        facts: Option<&BranchFacts>,
8811        posting: bool,
8812    ) -> Result<Option<PrMessage>> {
8813        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8814        let head = match final_head.as_deref() {
8815            Some(h) => crate::pr_outcome::FinalHead::At(h),
8816            None => crate::pr_outcome::FinalHead::Unknown,
8817        };
8818        // Only a pull request that is about to be posted pays for a summary.
8819        let summarized = if posting {
8820            Some(self.summarize_change(winner, final_head.as_deref()).await)
8821        } else {
8822            None
8823        };
8824        // A busy shared resource is not a permanent cause: publishing an
8825        // "unavailable" description would stick (a later head move never
8826        // edits the body). Stop before anything is pushed or opened; the
8827        // run is `Blocked` and a resume asks the summary again.
8828        if self.state.resource_block.is_some() {
8829            self.state.event(
8830                "pr-summary",
8831                "a shared resource was busy; nothing was published, resume to summarize again",
8832            );
8833            return Ok(None);
8834        }
8835        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8836            anyhow::bail!(
8837                "the final commit moved during public description generation; retry before publishing"
8838            );
8839        }
8840        let note = match (&summarized, &self.state.change_summary) {
8841            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8842            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8843            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8844            (None, _) => ChangeNote::NotAsked,
8845        };
8846        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8847        if !posting {
8848            let identity = crate::scrub::Identity::current();
8849            return Ok(Some(PrMessage {
8850                title: crate::scrub::scrub(&pr.title, &identity),
8851                body: crate::scrub::scrub(&pr.body, &identity),
8852            }));
8853        }
8854        // This very text was already rejected and asked about: no second
8855        // judge call, rewrite or question, just carry the record on.
8856        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8857        if let Some(known) = self
8858            .state
8859            .github_text
8860            .clone()
8861            .filter(|g| g.fingerprint == raw_fp)
8862        {
8863            return self.settle_withheld(winner, &pr, known).await;
8864        }
8865        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8866        let mut violations =
8867            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8868        let identity = crate::scrub::Identity::current();
8869        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8870            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8871            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8872        {
8873            violations.push(crate::github_text::Violation::SensitiveData);
8874        }
8875        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8876        // only a language violation starts one; sensitive data alone goes
8877        // straight to the owner, with the redacted text as the fallback.
8878        let has_language = violations
8879            .iter()
8880            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8881        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8882            self.state.event(
8883                "github-text",
8884                if has_language {
8885                    format!("description rejected: {violations:?}; requesting one rewrite")
8886                } else {
8887                    format!("description rejected: {violations:?}; asking the owner")
8888                },
8889            );
8890            let mut rewritten = false;
8891            let mut rejected_rewrite = None;
8892            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8893            {
8894                let output = text;
8895                if let Some(title) = summary_title(&output) {
8896                    // Judge the redacted candidate that will actually be posted.
8897                    // Sensitive data alone must not discard a useful rewrite.
8898                    let raw_body = summary_without_title(&output);
8899                    let clean_title = crate::scrub::scrub(&title, &identity);
8900                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8901                    let redacted = clean_title != title || body != raw_body;
8902                    let title = clean_title;
8903                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8904                    if !complete {
8905                        self.state.event(
8906                            "github-text",
8907                            "description rewrite omitted required findings",
8908                        );
8909                    }
8910                    if !body.trim().is_empty() {
8911                        body.push_str("\n\n");
8912                        body.push_str(&public_pr_evidence(&self.state, head));
8913                    }
8914                    let marker = format!("magi:run/{}", self.state.id);
8915                    if !body.trim().is_empty() && !body.contains(&marker) {
8916                        body.push_str(&format!("\n\n{marker}"));
8917                    }
8918                    let rewrite = PrMessage {
8919                        title: title.clone(),
8920                        body: body.clone(),
8921                    };
8922                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8923                    let mut failed = crate::github_text::check_for(
8924                        &self.state.config.graph,
8925                        &title,
8926                        &body,
8927                        redecision,
8928                    );
8929                    if (!crate::github_text::shareable(&title)
8930                        || !crate::github_text::shareable(&body))
8931                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8932                    {
8933                        failed.push(crate::github_text::Violation::SensitiveData);
8934                    }
8935                    if complete
8936                        && !title.trim().is_empty()
8937                        && !body.trim().is_empty()
8938                        && !failed.is_empty()
8939                    {
8940                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8941                            title: title.clone(),
8942                            body: body.clone(),
8943                            failed: failed
8944                                .iter()
8945                                .map(|v| crate::github_text::category(*v).to_owned())
8946                                .collect(),
8947                        });
8948                    }
8949                    if complete
8950                        && !title.trim().is_empty()
8951                        && !raw_body.trim().is_empty()
8952                        && failed.is_empty()
8953                    {
8954                        decision = redecision;
8955                        pr = PrMessage { title, body };
8956                        rewritten = true;
8957                        self.state
8958                            .event("github-text", "description rewrite passed");
8959                        if redacted {
8960                            self.state.event(
8961                                "github-text",
8962                                "sensitive data removed from description rewrite",
8963                            );
8964                        }
8965                    }
8966                }
8967            }
8968            if !rewritten {
8969                self.state.event(
8970                    "github-text",
8971                    "description rewrite unavailable or rejected; asking the owner",
8972                );
8973                if let Some(w) =
8974                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8975                {
8976                    // A different text than the one asked about before: that
8977                    // question no longer describes anything.
8978                    self.retire_github_text_question("the withheld text changed");
8979                    let record = crate::run::GithubTextAsk {
8980                        rewrite: rejected_rewrite,
8981                        fingerprint: raw_fp,
8982                        title: w.title,
8983                        body: w.body,
8984                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8985                        question: None,
8986                        asks: 0,
8987                        resolved: false,
8988                        chosen_title: None,
8989                        choice: None,
8990                        publication: None,
8991                    };
8992                    return self.settle_withheld(winner, &pr, record).await;
8993                }
8994            }
8995        }
8996        let (title, body) =
8997            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8998        Ok(Some(PrMessage { title, body }))
8999    }
9000
9001    /// Describe the whole change from its final diff in either run mode.
9002    ///
9003    /// The summary is tied to the final commit and to the merge base of the
9004    /// resolved remote base; a record for any other pair is dropped and the
9005    /// agent asked again, so a description never outlives the diff it was
9006    /// written from. The seat is read-only and the prompt tells it to read the
9007    /// diff and the code around it. `[roles] synthesizer` selects the agents
9008    /// (unset: the default pick), each asked at most once, advancing on an
9009    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
9010    /// carries the reason it is unavailable; nothing is invented then.
9011    async fn summarize_change(
9012        &mut self,
9013        winner: &Candidate,
9014        head: Option<&str>,
9015    ) -> std::result::Result<(), String> {
9016        let Some(head) = head else {
9017            return Err("the final commit could not be read".to_owned());
9018        };
9019        let remote = self.state.config.merge.remote.clone();
9020        let tracking = format!("{remote}/{}", self.state.base_branch);
9021        if !git::rev_exists(&winner.worktree, &tracking).await {
9022            return Err(format!("the remote base `{tracking}` could not be read"));
9023        }
9024        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
9025            Ok(b) if !b.is_empty() => b,
9026            _ => return Err("the merge base with the remote base could not be read".to_owned()),
9027        };
9028        let evidence = crate::pr_outcome::summary_evidence(
9029            &self.state,
9030            crate::pr_outcome::FinalHead::At(head),
9031        );
9032        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
9033        if self
9034            .state
9035            .change_summary
9036            .as_ref()
9037            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
9038        {
9039            return Ok(());
9040        }
9041        if self.state.change_summary.take().is_some() {
9042            self.state.event(
9043                "pr-summary",
9044                "the final diff or evidence changed; the earlier change summary was discarded",
9045            );
9046        }
9047        // Save the source snapshot even if no generation agent is available.
9048        if let Err(e) = write_artifact(
9049            &self.state,
9050            &format!("public-description-{fingerprint}.md"),
9051            &format!("Final diff: {base}..{head}\n\n{evidence}"),
9052        ) {
9053            tracing::warn!("could not save public description evidence: {e:#}");
9054        }
9055        let chain = match agent::pick_chain(
9056            &self.state.config.agents,
9057            self.state.config.roles.synthesizer.as_ref(),
9058            &agent::installed,
9059            "synthesizer",
9060        ) {
9061            Ok(chain) => chain,
9062            Err(e) => {
9063                tracing::warn!("no agent for the change summary: {e:#}");
9064                self.state.event("pr-summary", "no summary agent available");
9065                return Err("no agent was available to summarize it".to_owned());
9066            }
9067        };
9068        let subjects = git::subjects(&winner.worktree, &base, head)
9069            .await
9070            .unwrap_or_default();
9071        let stat = git::diff_stat(&winner.worktree, &base, head)
9072            .await
9073            .unwrap_or_default();
9074        let stat: String = stat.chars().take(6_000).collect();
9075        let mut prompt = crate::prompt::change_summary_in(
9076            &base,
9077            head,
9078            &subjects,
9079            stat.trim(),
9080            &self.state.config.graph.github_language,
9081        );
9082        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
9083        let sessions = self.state.config.graph.sessions;
9084        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
9085        let prompts = self.state.config.prompts.clone();
9086        let cache = self.state.config.cache_dir();
9087        let run = self.state.id.clone();
9088        for (i, spec) in chain.into_iter().enumerate() {
9089            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
9090            let github_spec = match agent::for_github_text(
9091                &spec,
9092                &self.state.config.graph.github_language,
9093                &winner.worktree,
9094            ) {
9095                Ok(spec) => spec,
9096                Err(e) => {
9097                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9098                    continue;
9099                }
9100            };
9101            let job = SeatJob {
9102                spec: github_spec,
9103                seat,
9104                cwd: winner.worktree.clone(),
9105                prompt: prompt.clone(),
9106                timeout,
9107                allow_write: false,
9108                sessions,
9109                artifacts: agent::artifacts_dir(&self.state.dir()),
9110                stem: if i == 0 {
9111                    "change-summary".to_owned()
9112                } else {
9113                    format!("change-summary-{}", spec.id)
9114                },
9115                handover: None,
9116            };
9117            let ctx = WaveCtx {
9118                run: &run,
9119                node: "pr-summary",
9120                prompts: &prompts,
9121                cache: cache.as_deref(),
9122                round: None,
9123                carry_seats: false,
9124                pauses: Some((&self.pause, &self.interrupt)),
9125            };
9126            let (seat, outcome) =
9127                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9128            self.state.seats.insert(seat.key.clone(), seat);
9129            let text = match outcome {
9130                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
9131                AgentOutcome::ResourceBlocked(why) => {
9132                    // No agent was asked: the run stops resumably before
9133                    // anything is published (see `guarded_pr_message`).
9134                    if self.state.resource_block.is_none() {
9135                        self.state.resource_block = Some(crate::run::ResourceBlock {
9136                            cause: crate::run::ResourceBlockCause::Other,
9137                            node: "pr-summary".to_owned(),
9138                            kind: "shared resource".to_owned(),
9139                            detail: why,
9140                            at: Timestamp::now(),
9141                        });
9142                    }
9143                    return Err("a shared resource was busy".to_owned());
9144                }
9145                _ => continue,
9146            };
9147            let blind = &self.state.config.blind;
9148            let Some(title) = summary_title(&text) else {
9149                continue;
9150            };
9151            let body: String = summary_without_title(&text)
9152                .lines()
9153                .filter(|l| !l.contains("magi:run/"))
9154                .collect::<Vec<_>>()
9155                .join("\n");
9156            let body = blind::sanitize_prose(body.trim(), blind);
9157            let title = blind::sanitize_prose(&title, blind);
9158            if body.trim().is_empty() || title.trim().is_empty() {
9159                continue;
9160            }
9161            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
9162                continue;
9163            }
9164            // The branch must still be the one the agent was shown.
9165            if git::rev_parse(&winner.worktree, "HEAD")
9166                .await
9167                .ok()
9168                .as_deref()
9169                != Some(head)
9170            {
9171                return Err("the branch moved while the summary was written".to_owned());
9172            }
9173            if git::merge_base(&winner.worktree, &tracking, head)
9174                .await
9175                .ok()
9176                .as_deref()
9177                != Some(base.as_str())
9178                || crate::pr_outcome::summary_evidence(
9179                    &self.state,
9180                    crate::pr_outcome::FinalHead::At(head),
9181                ) != evidence
9182            {
9183                return Err("the summary evidence changed while it was written".to_owned());
9184            }
9185            self.state.change_summary = Some(crate::run::ChangeSummary {
9186                head: head.to_owned(),
9187                base,
9188                evidence: fingerprint,
9189                title,
9190                body,
9191            });
9192            self.state.event(
9193                "pr-summary",
9194                format!("change summary written by {} for the final diff", spec.id),
9195            );
9196            if let Err(e) = self.state.save() {
9197                tracing::warn!("could not save the change summary: {e:#}");
9198            }
9199            return Ok(());
9200        }
9201        self.state
9202            .event("pr-summary", "no agent produced a change summary");
9203        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9204    }
9205
9206    /// One read-only rewrite request for a withheld description. The
9207    /// implementer's own conversation is tried first when it exists and can be
9208    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9209    /// writes it in a separate conversation, skipping the implementer's agent.
9210    /// The prompt carries the whole text, so no agent needs any memory. Each
9211    /// agent is asked at most once and the chain advances on an error, quota,
9212    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9213    /// that has run out of quota must not end the rewrite. Nothing is recorded
9214    /// as a handover.
9215    async fn rewrite_pr_text(
9216        &mut self,
9217        winner: &Candidate,
9218        pr: &PrMessage,
9219        violations: &[crate::github_text::Violation],
9220    ) -> Option<String> {
9221        let prompt = format!(
9222            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9223            pr.title, pr.body
9224        );
9225        let sessions = self.state.config.graph.sessions;
9226        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9227        // (spec, the resumed implementer's seat). A fallback seat is created
9228        // only when its turn comes, so an unused fallback leaves no record.
9229        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9230        let key = format!("impl-{}", winner.label);
9231        if let Some(spec) = self
9232            .state
9233            .config
9234            .agents
9235            .iter()
9236            .find(|a| a.id == winner.agent)
9237            .cloned()
9238            && let Some(existing) = self.state.seats.get(&key)
9239            && existing.agent == spec.id
9240            && agent::has_session(spec.kind, existing, sessions)
9241        {
9242            attempts.push((spec, Some(existing.clone())));
9243        }
9244        match agent::pick_chain(
9245            &self.state.config.agents,
9246            self.state.config.roles.language_judge.as_ref(),
9247            &agent::installed,
9248            "language_judge",
9249        ) {
9250            Ok(chain) => {
9251                for spec in chain {
9252                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9253                        continue;
9254                    }
9255                    attempts.push((spec, None));
9256                }
9257            }
9258            Err(e) if attempts.is_empty() => {
9259                tracing::warn!("no agent for the description rewrite: {e:#}");
9260                self.state
9261                    .event("github-text", "no rewrite agent available");
9262                return None;
9263            }
9264            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9265        }
9266        let prompts = self.state.config.prompts.clone();
9267        let cache = self.state.config.cache_dir();
9268        let run = self.state.id.clone();
9269        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9270            let resumed = resumed_seat.is_some();
9271            let seat = resumed_seat.unwrap_or_else(|| {
9272                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9273            });
9274            let github_spec = match agent::for_github_text(
9275                &spec,
9276                &self.state.config.graph.github_language,
9277                &winner.worktree,
9278            ) {
9279                Ok(spec) => spec,
9280                Err(e) => {
9281                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9282                    continue;
9283                }
9284            };
9285            let job = SeatJob {
9286                spec: github_spec,
9287                seat,
9288                cwd: winner.worktree.clone(),
9289                prompt: prompt.clone(),
9290                timeout: retry_budget(timeout_full, resumed),
9291                allow_write: false,
9292                sessions,
9293                artifacts: agent::artifacts_dir(&self.state.dir()),
9294                stem: if i == 0 {
9295                    "github-text-rewrite".to_owned()
9296                } else {
9297                    format!("github-text-rewrite-{}", spec.id)
9298                },
9299                handover: None,
9300            };
9301            let ctx = WaveCtx {
9302                run: &run,
9303                node: "github-text",
9304                prompts: &prompts,
9305                cache: cache.as_deref(),
9306                round: None,
9307                carry_seats: false,
9308                pauses: Some((&self.pause, &self.interrupt)),
9309            };
9310            let (seat, outcome) =
9311                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9312            self.state.seats.insert(seat.key.clone(), seat);
9313            match outcome {
9314                AgentOutcome::Ok(o)
9315                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9316                {
9317                    return Some(o.text);
9318                }
9319                AgentOutcome::ResourceBlocked(_) => return None,
9320                _ => {}
9321            }
9322        }
9323        None
9324    }
9325
9326    /// Abandon the standing posting-gate question, if any.
9327    fn retire_github_text_question(&mut self, why: &str) {
9328        let Some(id) = self
9329            .state
9330            .github_text
9331            .as_ref()
9332            .and_then(|g| g.question.clone())
9333        else {
9334            return;
9335        };
9336        let _ = ask::Questions::open().update(&id, |q| {
9337            q.abandon(why);
9338            Ok(())
9339        });
9340    }
9341
9342    /// Carry a withheld text to its end: file the question, park while it is
9343    /// open, and on an answer, silence or timeout pick the text to post. The
9344    /// decision is saved before it is used, so it applies once.
9345    async fn settle_withheld(
9346        &mut self,
9347        winner: &Candidate,
9348        pr: &PrMessage,
9349        mut g: crate::run::GithubTextAsk,
9350    ) -> Result<Option<PrMessage>> {
9351        use crate::github_text as gt;
9352        let store = ask::Questions::open();
9353        let timeout = self.state.config.graph.answer_timeout;
9354        if !g.resolved {
9355            let mut standing = g
9356                .question
9357                .as_ref()
9358                .and_then(|id| store.list_for_effects().into_iter().find(|q| &q.id == id));
9359            if let Some(q) = standing.as_ref()
9360                && q.status.open()
9361                && !store.held_reply(&q.id)
9362                && gt::expired(q, timeout)
9363            {
9364                let why = format!("no answer within {}s of asking", timeout.max(1));
9365                standing = store
9366                    .update(&q.id, |q| {
9367                        q.abandon(&why);
9368                        Ok(())
9369                    })
9370                    .ok()
9371                    .map(|(q, ())| q);
9372            }
9373            let mut ask_again: Option<Option<Vec<String>>> = None;
9374            match standing {
9375                // Recorded but never written (a stop between the two saves),
9376                // or removed: ask again without spending the retry.
9377                None if g.asks <= 1 => {
9378                    g.asks = 0;
9379                    ask_again = Some(None);
9380                }
9381                None => {
9382                    g.resolved = true;
9383                    g.chosen_title = None;
9384                    g.choice = Some(GithubTextChoice::Fallback);
9385                }
9386                Some(q) if q.status.open() => {
9387                    self.park_on_github_text(&g)?;
9388                    return Ok(None);
9389                }
9390                Some(q) => match gt::read_reply(&q) {
9391                    gt::Reply::Fallback => {
9392                        g.resolved = true;
9393                        g.chosen_title = None;
9394                        g.choice = Some(GithubTextChoice::Fallback);
9395                    }
9396                    gt::Reply::Original => {
9397                        g.resolved = true;
9398                        g.chosen_title = None;
9399                        g.choice = Some(GithubTextChoice::Original);
9400                    }
9401                    gt::Reply::Rewrite => {
9402                        g.resolved = true;
9403                        g.chosen_title = None;
9404                        g.choice = Some(if g.rewrite.is_some() {
9405                            GithubTextChoice::Rewrite
9406                        } else {
9407                            GithubTextChoice::Fallback
9408                        });
9409                    }
9410                    gt::Reply::Title(text) => {
9411                        let first = text
9412                            .lines()
9413                            .map(str::trim)
9414                            .find(|l| !l.is_empty())
9415                            .unwrap_or("");
9416                        let decision = if first.is_empty() {
9417                            None
9418                        } else {
9419                            let candidate = PrMessage {
9420                                title: first.to_owned(),
9421                                body: String::new(),
9422                            };
9423                            self.judge_pr_language(&winner.worktree, &candidate).await
9424                        };
9425                        match gt::vet_title(
9426                            &text,
9427                            decision,
9428                            self.state.config.graph.github_english(),
9429                        ) {
9430                            Ok(title) => {
9431                                g.resolved = true;
9432                                g.chosen_title = Some(title);
9433                                g.choice = Some(GithubTextChoice::MyTitle);
9434                            }
9435                            Err(cats) if g.asks < 2 => {
9436                                ask_again = Some(Some(
9437                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9438                                ))
9439                            }
9440                            Err(_) => {
9441                                g.resolved = true;
9442                                g.chosen_title = None;
9443                                g.choice = Some(GithubTextChoice::Fallback);
9444                            }
9445                        }
9446                    }
9447                },
9448            }
9449            if let Some(rejected) = ask_again {
9450                // A retry (after a rejected replacement title) is still about
9451                // the original text: the same ORIGINAL / FALLBACK comparison.
9452                let w = gt::Withheld {
9453                    title: g.title,
9454                    body: g.body,
9455                    categories: g.categories.clone(),
9456                };
9457                // Named by the original text's fingerprint alone, so a resume
9458                // rewrites the same file and the detail stays identical.
9459                let artifact = match crate::run::write_artifact(
9460                    &self.state,
9461                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9462                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9463                ) {
9464                    Ok(p) => Some(p),
9465                    Err(e) => {
9466                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9467                        None
9468                    }
9469                };
9470                let (detail, comparison) = gt::question_content(
9471                    &w,
9472                    &pr.title,
9473                    &pr.body,
9474                    &self.state.id,
9475                    rejected.as_deref(),
9476                    artifact.as_deref(),
9477                    g.rewrite.as_ref(),
9478                );
9479                let mut q = ask::Question::new(
9480                    self.state.id.clone(),
9481                    gt::ASK_NODE.to_owned(),
9482                    gt::ASK_SEAT.to_owned(),
9483                    gt::question_summary(&self.state.config.graph.language, &w),
9484                    detail,
9485                    gt::question_choices(&w, g.rewrite.as_ref()),
9486                );
9487                q.posting_comparison = Some(comparison);
9488                q.answer_timeout = timeout;
9489                // Recorded before it is written: a stop in between leaves a
9490                // record naming a question the resume then files again.
9491                g.question = Some(q.id.clone());
9492                g.asks += 1;
9493                self.state.github_text = Some(g.clone());
9494                self.state.event(
9495                    "github-text",
9496                    format!("asking the owner about the withheld text ({})", q.short()),
9497                );
9498                self.state.save()?;
9499                store
9500                    .put_executable(&mut q)
9501                    .context("file the posting-gate question")?;
9502                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9503                    tracing::warn!(
9504                        "could not notify about posting-gate question {}: {e:#}",
9505                        q.short()
9506                    );
9507                }
9508                self.park_on_github_text(&g)?;
9509                return Ok(None);
9510            }
9511        }
9512        let resolved = g.chosen_title.clone();
9513        let original = g.choice == Some(GithubTextChoice::Original);
9514        // Field names only: the event log never records the text.
9515        self.state.event(
9516            "github-text",
9517            if original {
9518                format!(
9519                    "posting the original text for: {}",
9520                    match (g.title, g.body) {
9521                        (true, true) => "title, description",
9522                        (true, false) => "title",
9523                        _ => "description",
9524                    }
9525                )
9526            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9527                "posting the saved rewrite title and description".to_owned()
9528            } else if resolved.is_some() {
9529                "posting the owner's replacement title".to_owned()
9530            } else {
9531                "posting the neutral text for the withheld fields".to_owned()
9532            },
9533        );
9534        self.state.github_text = Some(g.clone());
9535        self.state.save()?;
9536        if g.choice == Some(GithubTextChoice::Rewrite)
9537            && let Some(r) = g.rewrite.as_ref()
9538        {
9539            return Ok(Some(PrMessage {
9540                title: r.title.clone(),
9541                body: r.body.clone(),
9542            }));
9543        }
9544        let w = gt::Withheld {
9545            title: g.title,
9546            body: g.body,
9547            categories: g.categories.clone(),
9548        };
9549        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9550        if let Some(t) = resolved {
9551            title = t;
9552        }
9553        // Every withheld field is now neutral or vetted English.
9554        let english = gt::LanguageDecision {
9555            title_english: true,
9556            body_english: true,
9557            body_complete: true,
9558        };
9559        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9560        if original {
9561            // The owner's explicit choice overrides the gate for the fields it
9562            // was made for; the others keep what the gate produced.
9563            if g.title {
9564                title = pr.title.clone();
9565            }
9566            if g.body {
9567                body = pr.body.clone();
9568            }
9569        }
9570        Ok(Some(PrMessage { title, body }))
9571    }
9572
9573    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9574        self.state.github_text = Some(g.clone());
9575        self.state.end_pass_parked(ParkCause::Question);
9576        self.state.parked = true;
9577        self.state.event(
9578            "github-text",
9579            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9580        );
9581        self.state.save()
9582    }
9583
9584    /// Ask `[roles] language_judge` about this text and record which source
9585    /// decided; the text itself is never recorded.
9586    async fn judge_pr_language(
9587        &mut self,
9588        cwd: &Path,
9589        pr: &PrMessage,
9590    ) -> Option<crate::github_text::LanguageDecision> {
9591        self.state.config.roles.language_judge.as_ref()?;
9592        if !self.state.config.graph.language_gate() {
9593            return None;
9594        }
9595        let decision =
9596            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9597        self.state.event(
9598            "github-text",
9599            if decision.is_some() {
9600                "language decided by the language judge"
9601            } else {
9602                "language judge unavailable; using built-in heuristics"
9603            },
9604        );
9605        decision
9606    }
9607
9608    fn merge(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9609        Box::pin(self.merge_inner())
9610    }
9611
9612    async fn merge_inner(&mut self) -> Result<()> {
9613        if self.resource_halt()? || self.hold_base_sync().await? {
9614            return Ok(());
9615        }
9616        // Same reasoning as `gate`: ask the review and gate records directly
9617        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9618        // skip can rewrite on reentry to something that no longer says
9619        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9620        // so a hand-off (open findings, green verification) reaches merge
9621        // exactly like a genuinely clean round does.
9622        //
9623        // A run resumed mid-`land` never reaches here at all: `execute`
9624        // recognises `RunStatus::Landing` before it even calls `prep`, and
9625        // routes straight to `run_land` instead. That has to happen a level
9626        // up from this function, not with a check in here, because
9627        // `review_loop`'s own status recomputation (see its doc) runs
9628        // *before* `merge` on every reentry and would otherwise overwrite
9629        // the `Landing` marker with `Gating` before this node ever saw it.
9630        if self
9631            .state
9632            .base_sync
9633            .as_ref()
9634            .is_some_and(|s| s.conflict.is_some())
9635            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9636                != Some(RunStatus::Gating)
9637            // `gate_ran == false` is not "passed" - `gate` leaves it false
9638            // both before it has ever run and when its last attempt was
9639            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9640            // permission to merge on nothing but the review record. Only a
9641            // gate that actually ran may proceed; an intentional local
9642            // review can also finish with verification explicitly not configured.
9643            // `RunState::gate_status` computes that distinction.
9644            || !self.state.gate_allows_progress()
9645        {
9646            return Ok(());
9647        }
9648        // This node's own record, not `status`: `status == Ready` is not
9649        // unique to the harmless `MergeMode::None` path this line was
9650        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9651        // run's PR was closed without merging — and on that run `mode` is
9652        // still `Pr`, so a reentry that fell through here would push and
9653        // open a second pull request. `self.state.merge` is set exactly once
9654        // this node (or `land`) has already produced a verdict, under every
9655        // mode, which is what "already done" actually means here.
9656        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9657            return Ok(());
9658        }
9659        let Some(winner) = self.state.winner().cloned() else {
9660            return Ok(());
9661        };
9662        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9663        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9664            self.state.status = RunStatus::Blocked;
9665            self.state.event(
9666                "gate",
9667                "winner changed after gate; fresh verification required",
9668            );
9669            self.state.save()?;
9670            return Ok(());
9671        }
9672        let wait = self.repository_wait();
9673        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9674            return Ok(());
9675        }
9676        let repo = self.state.repo.clone();
9677        let base = self.state.base_branch.clone();
9678        let mode = self.state.config.merge.mode;
9679        let style = self.state.config.merge.style;
9680        let facts = if is_review_run(&self.state) {
9681            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9682            let start = review_base(
9683                &repo,
9684                &self.state.config.merge.remote,
9685                &base,
9686                &self.state.base_commit,
9687                &winner.branch,
9688            )
9689            .await;
9690            branch_facts(&repo, &start, &winner.branch).await
9691        } else {
9692            None
9693        };
9694        // `None` when the base could not be freshly read: then an adopted
9695        // pull request's title is left alone.
9696        let leaked = if is_review_run(&self.state) {
9697            leaked_subjects(&self.state, &winner.branch).await
9698        } else {
9699            Some(Vec::new())
9700        };
9701        // `None`: parked on the owner's word about a withheld title or body.
9702        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9703        // the resume arrives here again.
9704        let Some(pr) = self
9705            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9706            .await?
9707        else {
9708            return Ok(());
9709        };
9710        let message = pr.commit_message();
9711
9712        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9713        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9714            return Ok(());
9715        }
9716        let outcome = match mode {
9717            MergeMode::None => MergeOutcome {
9718                mode,
9719                ok: true,
9720                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9721                empty: false,
9722            },
9723            MergeMode::Pr | MergeMode::Local
9724                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9725            {
9726                MergeOutcome {
9727                    mode,
9728                    ok: false,
9729                    detail: empty_candidate_detail(&self.state, &base),
9730                    empty: true,
9731                }
9732            }
9733            MergeMode::Local => {
9734                let on = git::current_branch(&repo).await?;
9735                if on.as_deref() != Some(base.as_str()) {
9736                    MergeOutcome {
9737                        mode,
9738                        ok: false,
9739                        detail: format!(
9740                            "{} has {} checked out, not the base branch {base}",
9741                            repo.display(),
9742                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9743                        ),
9744                        empty: false,
9745                    }
9746                } else if !git::is_clean(&repo).await? {
9747                    MergeOutcome {
9748                        mode,
9749                        ok: false,
9750                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9751                        empty: false,
9752                    }
9753                } else {
9754                    let out = match style {
9755                        MergeStyle::Merge => {
9756                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9757                        }
9758                        MergeStyle::Squash => {
9759                            git::merge_squash(&repo, &winner.branch, &message).await?
9760                        }
9761                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9762                    };
9763                    MergeOutcome {
9764                        mode,
9765                        ok: out.ok(),
9766                        detail: if out.ok() { out.stdout } else { out.stderr },
9767                        empty: false,
9768                    }
9769                }
9770            }
9771            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9772                self.state.pr_publication_pending = true;
9773                self.state.save()?;
9774                let out =
9775                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9776                self.state.pr_publication_pending = out.is_err();
9777                MergeOutcome {
9778                    mode,
9779                    ok: out.is_ok(),
9780                    detail: match out {
9781                        Ok(url) => url,
9782                        Err(e) => format!("{e:#}"),
9783                    },
9784                    empty: false,
9785                }
9786            }
9787            MergeMode::Pr => {
9788                let remote = self.state.config.merge.remote.clone();
9789                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9790                let pushed = if self.state.foreign_checkout.is_some() {
9791                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9792                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9793                } else {
9794                    git::push(&winner.worktree, &remote, &winner.branch).await?
9795                };
9796                if !pushed.ok() {
9797                    MergeOutcome {
9798                        mode,
9799                        ok: false,
9800                        detail: pushed.stderr,
9801                        empty: false,
9802                    }
9803                } else {
9804                    if self.state.foreign_checkout.is_some() {
9805                        self.state.pr_pushed_head = Some(delivery_head.clone());
9806                    }
9807                    // A retry or resume of a run whose branch already has an
9808                    // open pull request adopts it rather than failing on a
9809                    // duplicate. Only this winner branch into this base:
9810                    // `branch_for` derives the name from the run id, so a
9811                    // different run's pull request never matches.
9812                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9813                    let out = match pr_merge_plan(found) {
9814                        PrPlan::Create => {
9815                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9816                                .await
9817                        }
9818                        PrPlan::Adopt { url, title } => {
9819                            self.state
9820                                .event("merge", format!("Pr: adopted open pull request {url}"));
9821                            if self.state.explicit_github_text_selection() {
9822                                // An explicit publication selection applies the prepared
9823                                // payload; acceptance alone is not publication proof.
9824                                land::set_pr_text_verbatim(
9825                                    &winner.worktree,
9826                                    &url,
9827                                    &pr.title,
9828                                    &pr.body,
9829                                )
9830                                .await
9831                                .map(|()| url)
9832                            } else {
9833                                if title != pr.title
9834                                    && (!is_review_run(&self.state)
9835                                        || leaked
9836                                            .as_deref()
9837                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9838                                    && let Err(e) =
9839                                        if self.state.github_text.as_ref().is_some_and(|g| {
9840                                            g.resolved
9841                                                && g.title
9842                                                && g.choice == Some(GithubTextChoice::Original)
9843                                        }) {
9844                                            // The owner chose to post this title unmodified.
9845                                            land::set_pr_title_verbatim(
9846                                                &winner.worktree,
9847                                                &url,
9848                                                &pr.title,
9849                                            )
9850                                            .await
9851                                        } else {
9852                                            land::set_pr_title(
9853                                                &mut self.state,
9854                                                &winner.worktree,
9855                                                &url,
9856                                                &pr.title,
9857                                            )
9858                                            .await
9859                                        }
9860                                {
9861                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9862                                    self.state
9863                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9864                                }
9865                                Ok(url)
9866                            }
9867                        }
9868                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9869                    };
9870                    match out {
9871                        Ok(url) => MergeOutcome {
9872                            mode,
9873                            ok: true,
9874                            detail: url,
9875                            empty: false,
9876                        },
9877                        Err(e) => MergeOutcome {
9878                            mode,
9879                            ok: false,
9880                            detail: e.to_string(),
9881                            empty: false,
9882                        },
9883                    }
9884                }
9885            }
9886        };
9887
9888        self.state.status = match (mode, outcome.ok) {
9889            (MergeMode::None, _) => RunStatus::Ready,
9890            (_, true) if outcome.empty => RunStatus::Ready,
9891            (MergeMode::Pr, true) => RunStatus::Ready,
9892            (_, true) => RunStatus::Merged,
9893            (_, false) => RunStatus::Blocked,
9894        };
9895        self.state.event(
9896            "merge",
9897            format!(
9898                "{:?}: {}",
9899                mode,
9900                outcome.detail.lines().next().unwrap_or("")
9901            ),
9902        );
9903        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9904            self.state.record_github_text_publication(&outcome.detail);
9905        }
9906        self.state.merge = Some(outcome);
9907        self.state.save()?;
9908
9909        // The PR is open and the run would historically stop here, leaving the
9910        // operator to watch checks, feed review comments back to a fixer, and
9911        // merge. That was done by hand six times in one session before this
9912        // existed. Opt-in, because merging is the one irreversible thing magi
9913        // can do to a repository.
9914        if self.state.config.graph.land
9915            && mode == MergeMode::Pr
9916            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9917        {
9918            self.run_land().await?;
9919        }
9920        // `run_land` may have left `status` at `Landing` - still waiting on
9921        // CI or the owner's approval, not actually settled - so this has to
9922        // read whatever `status` ended up as here, not the `Merged` this
9923        // function set a few lines up.
9924        self.settle_questions();
9925        Ok(())
9926    }
9927
9928    /// Enter `land`.
9929    ///
9930    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9931    /// a resumed run's re-entry. `land::land` itself is what serialises the
9932    /// two git-mutating moments inside the loop — the rebase push and
9933    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9934    /// to hold a lock across the whole call, and doing so would serialise
9935    /// this run's CI wait against a *different* run's land-approval resume
9936    /// in the same repository, which is exactly the "must not wait on
9937    /// another task" property the daemon's slot-freeing exists to give.
9938    fn run_land(&mut self) -> impl std::future::Future<Output = Result<()>> + '_ {
9939        Box::pin(self.run_land_inner())
9940    }
9941
9942    async fn run_land_inner(&mut self) -> Result<()> {
9943        crate::pr_head::ensure_owner(&self.state)?;
9944        let url = self
9945            .state
9946            .pr
9947            .as_ref()
9948            .map(|pr| pr.url.clone())
9949            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9950            .unwrap_or_default();
9951        let url = url.lines().next().unwrap_or("").trim().to_owned();
9952        if !url.starts_with("http") {
9953            return Ok(());
9954        }
9955        // A land failure is not a lost run: the work is on a branch and the
9956        // pull request is open, which is exactly where a human takes over.
9957        loop {
9958            match land::land(&mut self.state, &url).await {
9959                Ok(_)
9960                    if self
9961                        .state
9962                        .merge_repairs
9963                        .last()
9964                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9965                {
9966                    if self.repair_merge_review().await? {
9967                        continue;
9968                    }
9969                }
9970                Ok(_) if self.state.resource_block.is_some() => {
9971                    return self.stop_for_resource();
9972                }
9973                Ok(pr) if self.state.parked => {
9974                    // `land` already saved the parked marker; nothing here
9975                    // overrides `status` back to a terminal value while an
9976                    // approval is still outstanding.
9977                    let _ = pr;
9978                }
9979                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9980                    self.state.status = RunStatus::Superseded;
9981                    self.state.save()?;
9982                }
9983                Ok(pr) => {
9984                    self.state.status = match pr.state {
9985                        land::PrLifecycle::Merged => RunStatus::Merged,
9986                        _ => RunStatus::Blocked,
9987                    };
9988                    // Downstream of a confirmed merge only - see
9989                    // `bump::should_release_bump`'s own doc for why this one
9990                    // check covers all three of `land`'s success paths.
9991                    // Best-effort: the run already landed, so a failure here
9992                    // (the decision call, `gh`, `cargo`) is recorded and never
9993                    // turns a landed run into a failed one.
9994                    if bump::should_release_bump(self.state.status)
9995                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9996                    {
9997                        // The event is the run's own record. Not-eligible cases
9998                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9999                        // `Err` is a bump that was tried and failed:
10000                        // `after_merge` itself raises the operator notice for
10001                        // that, whether or not a release PR exists yet.
10002                        self.state
10003                            .event("bump", format!("release bump skipped: {e:#}"));
10004                    }
10005                    // Independent of the bump, and best-effort in the same way:
10006                    // findings the merge left open become follow-up tasks.
10007                    if self.state.status == RunStatus::Merged {
10008                        crate::followup::after_merge(&mut self.state, &pr.url).await;
10009                    }
10010                    self.state.save()?;
10011                }
10012                Err(e) => {
10013                    if e.is::<crate::repo_lock::ResourceBlocked>() {
10014                        return Err(e);
10015                    }
10016                    self.state.status = RunStatus::Blocked;
10017                    self.state.event("land", format!("gave up: {e}"));
10018                    self.state.save()?;
10019                }
10020            }
10021            break;
10022        }
10023        Ok(())
10024    }
10025
10026    /// One repair cycle stays under the existing driver/task claim. An
10027    /// interrupted fixer is rolled back and held rather than run twice.
10028    async fn repair_merge_review(&mut self) -> Result<bool> {
10029        use crate::run::MergeRepairPhase as Phase;
10030        crate::pr_head::ensure_owner(&self.state)?;
10031        let i = self.state.merge_repairs.len() - 1;
10032        let request = self.state.merge_repairs[i].clone();
10033        if request.phase == Phase::Fixing {
10034            let restored = match self.state.winner().cloned() {
10035                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
10036                None => Err(anyhow::anyhow!("repair has no winning branch")),
10037            };
10038            let why = match restored {
10039                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
10040                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
10041                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
10042            };
10043            self.hold_merge_repair(i, why)?;
10044            return Ok(false);
10045        }
10046        // A completed repair gate is a publication checkpoint. On reentry,
10047        // reject external changes before the ordinary gate path can replace
10048        // that evidence by verifying a new head. Requested repairs still
10049        // start their own review generation below.
10050        if request.phase == Phase::Reviewing && self.state.gate_ran {
10051            let winner = self
10052                .state
10053                .winner()
10054                .context("repair has no winning branch")?;
10055            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
10056            if self.state.gate_head.as_deref() != Some(head.as_str())
10057                || !git::is_clean(&winner.worktree).await?
10058            {
10059                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10060                return Ok(false);
10061            }
10062        }
10063        let failure = if i >= self.state.config.graph.land_rounds
10064            || self.state.config.graph.review_rounds == 0
10065        {
10066            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
10067        } else {
10068            None
10069        };
10070        if let Some(why) = failure {
10071            self.hold_merge_repair(i, why)?;
10072            return Ok(false);
10073        }
10074        if request.phase == Phase::Requested {
10075            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
10076                Ok(pr) => pr,
10077                Err(e) => {
10078                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
10079                    return Ok(false);
10080                }
10081            };
10082            self.state.merge_repairs[i].phase = Phase::Fixing;
10083            self.state.land_approval = None;
10084            self.state.land_armed_head = None;
10085            self.state.save()?;
10086            // Serialize the exact records, including finding ids/details,
10087            // reconsideration reasons, final votes and verification output.
10088            let evidence = serde_json::to_string_pretty(&request)?;
10089            let reason = format!(
10090                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
10091            );
10092            let budget = self.state.config.graph.land_rounds;
10093            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
10094            match result {
10095                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
10096                Ok(land::Fixed::Deferred) => {
10097                    // An unavailable fixer did not spend this repair request.
10098                    // Restore partial fallback work before retrying it, as for
10099                    // an interrupted repair, without closing the request.
10100                    let worktree = self
10101                        .state
10102                        .winner()
10103                        .context("repair has no winning branch")?
10104                        .worktree
10105                        .clone();
10106                    self.graft_restore(&worktree, &request.head).await?;
10107                    self.state.merge_repairs[i].phase = Phase::Requested;
10108                    self.stop_for_resource()?;
10109                    return Ok(false);
10110                }
10111                Ok(land::Fixed::Failed(why)) => {
10112                    self.hold_merge_repair(
10113                        i,
10114                        format!(
10115                            "repair failed: {why}; inspect the branch and run an explicit review"
10116                        ),
10117                    )?;
10118                    return Ok(false);
10119                }
10120                Err(e) => {
10121                    self.hold_merge_repair(
10122                        i,
10123                        format!(
10124                            "repair failed: {e:#}; inspect the branch and run an explicit review"
10125                        ),
10126                    )?;
10127                    return Ok(false);
10128                }
10129            }
10130            // Even no change gets a real new reviewer wave. The previous
10131            // evidence remains in the request; a former clean/concluded round
10132            // cannot short-circuit the new review or gate.
10133            self.begin_merge_repair_review(i)?;
10134        }
10135        self.review_loop().await?;
10136        self.gate().await?;
10137        if self.resource_halt()? || self.state.parked {
10138            return Ok(false);
10139        }
10140        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
10141            != Some(RunStatus::Gating)
10142            || !self.state.gate_allows_progress()
10143        {
10144            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
10145            return Ok(false);
10146        }
10147        // Review and gate may have added commits after the initial repair.
10148        // Publish exactly their final head to the existing PR, with a lease.
10149        let winner = self
10150            .state
10151            .winner()
10152            .cloned()
10153            .context("repair has no winning branch")?;
10154        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
10155        if self.state.gate_head.as_deref() != Some(after.as_str())
10156            || !git::is_clean(&winner.worktree).await?
10157        {
10158            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
10159            return Ok(false);
10160        }
10161        let expected = self
10162            .state
10163            .pr_pushed_head
10164            .as_deref()
10165            .unwrap_or(&request.head)
10166            .to_owned();
10167        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
10168        // Push the checked commit itself: a local ref moving after the check
10169        // must not change which commit the forge receives.
10170        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
10171            return Ok(false);
10172        }
10173        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
10174        let refspec = format!("{after}:refs/heads/{public}");
10175        let push = git::git_raw(
10176            &winner.worktree,
10177            &["push", &lease, &self.state.config.merge.remote, &refspec],
10178        )
10179        .await;
10180        match push {
10181            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
10182            Ok(out) => {
10183                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
10184                return Ok(false);
10185            }
10186            Err(e) => {
10187                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
10188                return Ok(false);
10189            }
10190        }
10191        self.state.merge_repairs[i].phase = Phase::Complete;
10192        self.state.status = RunStatus::Landing;
10193        self.state.event(
10194            "repair",
10195            "new review and gate completed; fresh owner merge decision required",
10196        );
10197        self.state.save()?;
10198        Ok(true)
10199    }
10200
10201    fn merge_repair_artifacts(&self) -> PathBuf {
10202        let artifacts = agent::artifacts_dir(&self.state.dir());
10203        match self
10204            .state
10205            .merge_repairs
10206            .last()
10207            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10208        {
10209            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10210            None => artifacts,
10211        }
10212    }
10213
10214    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10215        if let Some(e2e) = self.state.head_e2e.take() {
10216            self.state
10217                .verification_history
10218                .push(crate::run::VerificationEvidence {
10219                    superseded_by: None,
10220                    archived_at: Some(Timestamp::now()),
10221                    fixes_spent: Some(self.state.gate_fixes.len()),
10222                    gate_head: None,
10223                    gate_ran: false,
10224                    gate: Vec::new(),
10225                    e2e: Some(e2e),
10226                });
10227        }
10228        self.state.reviews.clear();
10229        self.state.gate.clear();
10230        self.state.gate_ran = false;
10231        self.state.gate_head = None;
10232        self.state.gate_fixes.clear();
10233        self.state.gate_repair_e2e = None;
10234        self.state.pre_gate.clear();
10235        self.state.pre_gate_commit = None;
10236        self.state.contested_handoff = None;
10237        self.state.land_reruns = None;
10238        self.state.land_approval = None;
10239        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10240        self.state.status = RunStatus::Reviewing;
10241        self.state.save()
10242    }
10243
10244    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10245        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10246        self.state.merge_repairs[index].failure = Some(why.clone());
10247        self.state.status = RunStatus::Blocked;
10248        self.state.event("repair", why);
10249        self.state.save()
10250    }
10251
10252    // -------------------------------------------------------------- helpers
10253
10254    /// Fetch or create a seat, keeping its conversation across nodes.
10255    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10256        if let Some(existing) = self.state.seats.get(key)
10257            && existing.agent == agent
10258        {
10259            return existing.clone();
10260        }
10261        // A seat that changes agent mints its session id from the agent too,
10262        // like a handover: the old agent's uuid is already taken by the CLI.
10263        let fresh = if self.state.seats.contains_key(key) {
10264            handover_seat(key, agent, self.state.next_seat_seed())
10265        } else {
10266            SeatState::new(key, agent, self.state.seed)
10267        };
10268        self.state.seats.insert(key.to_owned(), fresh.clone());
10269        fresh
10270    }
10271
10272    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10273    /// seat to another roster agent, in which case that agent. Nodes that
10274    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10275    /// reconsideration) must keep talking to whoever answered it, not slip
10276    /// back to the agent that failed it.
10277    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10278        match self.state.seats.get(key) {
10279            Some(s) if s.agent != spec.id => {
10280                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10281            }
10282            _ => spec,
10283        }
10284    }
10285
10286    /// A candidate rendered for judging, with the leak policy applied.
10287    fn view(&self, c: &Candidate) -> CandidateView {
10288        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10289            .unwrap_or_default();
10290        let (patch, _) = blind::sanitize_patch(
10291            &format!("candidate {} patch", c.label),
10292            &raw,
10293            &self.state.config.blind,
10294        );
10295        CandidateView {
10296            label: c.label,
10297            branch: c.branch.clone(),
10298            summary: c.summary.clone(),
10299            stat: c.stat.clone(),
10300            patch,
10301        }
10302    }
10303
10304    /// The full candidate set as prompt text, for seats with no live session.
10305    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10306        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10307        prompt::judge(
10308            "(see above)",
10309            &views,
10310            self.roles.judges.len(),
10311            base_short,
10312            "en",
10313        )
10314    }
10315
10316    /// The final-vote prompt with everything a seat that has no session of its
10317    /// own needs: the candidates, the seat's own ranking and reasons, and the
10318    /// anonymised deliberation it took part in (only when there was one, so a
10319    /// handed-over seat never sees more than the seat it replaces did). The
10320    /// `Final vote` heading stays first.
10321    fn vote_prompt_full(
10322        &self,
10323        j: usize,
10324        labels: &[char],
10325        language: &str,
10326        candidates: &[Candidate],
10327        base_short: &str,
10328    ) -> String {
10329        let mut text = format!(
10330            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10331            prompt::final_vote(labels, language),
10332            self.state.instruction,
10333            self.candidate_block(candidates, base_short)
10334        );
10335        if let Some(own) = self
10336            .state
10337            .judgements
10338            .get(j)
10339            .filter(|r| !r.ranking.is_empty())
10340        {
10341            let reasons = own
10342                .reasons
10343                .iter()
10344                .map(|(k, v)| format!("- {k}: {v}"))
10345                .collect::<Vec<_>>()
10346                .join("\n");
10347            text.push_str(&format!(
10348                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10349                own.ranking.iter().collect::<String>(),
10350                if reasons.is_empty() {
10351                    ""
10352                } else {
10353                    ", because:\n"
10354                }
10355            ));
10356        }
10357        if !self.state.deliberation.is_empty() {
10358            text.push_str("\n# What was argued before this vote\n");
10359            for t in self.transcript(&[], j) {
10360                text.push_str(&format!(
10361                    "\n## {}{}\n\n{}\n",
10362                    t.who,
10363                    if t.is_self { " (you)" } else { "" },
10364                    t.body.trim()
10365                ));
10366            }
10367        }
10368        text
10369    }
10370
10371    /// Anonymised transcript for judge `self_idx`.
10372    ///
10373    /// The initial rankings are always the opening statements. Seeding them
10374    /// only when no turn had been taken yet meant every judge after the first
10375    /// argued against a single voice instead of against the actual split — the
10376    /// disagreement is the information, so it is always on the table.
10377    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10378        let mut turns = Vec::new();
10379        for j in &self.state.judgements {
10380            if j.ranking.is_empty() {
10381                continue;
10382            }
10383            let reasons = j
10384                .reasons
10385                .iter()
10386                .map(|(k, v)| format!("- {k}: {v}"))
10387                .collect::<Vec<_>>()
10388                .join("\n");
10389            turns.push(Turn {
10390                who: format!("Judge {} (opening ranking)", j.judge),
10391                is_self: j.judge == self_idx + 1,
10392                body: format!(
10393                    "Ranked {}{}{reasons}",
10394                    j.ranking.iter().collect::<String>(),
10395                    if reasons.is_empty() {
10396                        ""
10397                    } else {
10398                        ", because:\n"
10399                    }
10400                ),
10401            });
10402        }
10403        for t in self
10404            .state
10405            .deliberation
10406            .iter()
10407            .flat_map(|r| r.turns.iter())
10408            .chain(current)
10409        {
10410            turns.push(Turn {
10411                who: format!("Judge {}", t.judge),
10412                is_self: t.judge == self_idx + 1,
10413                body: t.body.clone(),
10414            });
10415        }
10416        turns
10417    }
10418}
10419
10420/// Does this seat still hold the context a follow-up prompt would rely on?
10421fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10422    agent::has_session(spec.kind, seat, sessions)
10423}
10424
10425/// The next entry in `roster` after `start`, never wrapping back to the
10426/// front, whose id is not in `tried` yet.
10427///
10428/// Starts one past `start` rather than at the front of `roster`: `start` is
10429/// the seat's own original position, and a seat whose candidate slot already
10430/// sits on the roster's second entry must fall through to the third next, not
10431/// restart at the first — which is very likely a different candidate's own
10432/// agent already. Never wraps back past `start`, for the same reason: an
10433/// entry earlier in the roster than the seat's own position is almost
10434/// certainly some *other* candidate slot's own agent, and once the tail of
10435/// the roster is exhausted there are no more untried agents for *this* seat
10436/// to fall through to — the caller's fallback chain ends there, exactly as
10437/// "no further untried agents remain in the list for that seat" asks for.
10438///
10439/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10440/// the same id twice (an operator's `roles.implementers` typo, or a
10441/// `[[agents]]` list reused across roles) must not let
10442/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10443/// over `roster` either finds an untried id or runs out, so this always
10444/// terminates regardless of duplicates.
10445fn next_untried_in_roster<'a>(
10446    roster: &'a [AgentSpec],
10447    start: usize,
10448    tried: &BTreeSet<String>,
10449) -> Option<&'a AgentSpec> {
10450    roster
10451        .get(start + 1..)?
10452        .iter()
10453        .find(|s| !tried.contains(&s.id))
10454}
10455
10456/// The successor for a seat, shared by all four seat kinds (implement, judge,
10457/// review, advise). `others` holds the ids that *currently* occupy the other
10458/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10459/// sees them), so roster entries beyond the seat count act as spares: a failed
10460/// seat goes to an agent no other seat holds whenever the roster permits.
10461///
10462/// `carried` is `Some` only for the review loop's carried failure history.
10463/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10464/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10465/// whole roster: untried and not another seat's occupant; (3) the plain walk
10466/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10467/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10468/// a better outcome than an empty seat, and it keeps the answer to "is there
10469/// a successor at all" exactly what it was before occupants were considered,
10470/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10471/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10472/// carried failure is a cheaper bet than doubling an agent on the panel, and
10473/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10474/// the same round are handled one at a time, so a seat later in the batch
10475/// sees an earlier one's new occupant but not yet its own freed agent.
10476fn pick_successor<'a>(
10477    roster: &'a [AgentSpec],
10478    start: usize,
10479    tried: &BTreeSet<String>,
10480    carried: Option<&BTreeSet<String>>,
10481    others: &BTreeSet<String>,
10482) -> Option<&'a AgentSpec> {
10483    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10484    roster
10485        .get(start + 1..)
10486        .and_then(|tail| {
10487            tail.iter()
10488                .filter(free)
10489                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10490        })
10491        .or_else(|| {
10492            carried
10493                .is_some()
10494                .then(|| roster.iter().find(free))
10495                .flatten()
10496        })
10497        .or_else(|| match carried {
10498            Some(c) => next_for_seat(roster, start, tried, c),
10499            None => next_untried_in_roster(roster, start, tried),
10500        })
10501}
10502
10503/// The next agent for a seat that carries its failure history across rounds
10504/// (the review loop). `round_tried` is this round's own bound and starts
10505/// empty every round; `carried_failed` only decides priority.
10506///
10507/// First: an id walking forward from `start`, never wrapping, that is neither
10508/// tried this round nor failed in an earlier one. Only when that is exhausted
10509/// does it rescue: the first roster id (in roster order, so this one *does*
10510/// look before `start`) not yet tried this round, which is by then a carried
10511/// failure. Each id is rescued at most once per round, so it cannot loop.
10512fn next_for_seat<'a>(
10513    roster: &'a [AgentSpec],
10514    start: usize,
10515    round_tried: &BTreeSet<String>,
10516    carried_failed: &BTreeSet<String>,
10517) -> Option<&'a AgentSpec> {
10518    roster
10519        .get(start + 1..)?
10520        .iter()
10521        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10522        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10523}
10524
10525/// Where a reviewer seat starts a round: the agent that last answered it when
10526/// it is still on the roster and not marked failed, else the spec's own agent
10527/// unless it failed, else the next roster agent that has not failed, else the
10528/// spec's own agent again (the whole roster failed). Ids no longer on the
10529/// roster are ignored. An empty roster has no handover, so the spec stands.
10530fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10531    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10532        return spec;
10533    };
10534    let ok = |id: &str| !h.failed.contains(id);
10535    if let Some(last) = h.last_ok.as_deref()
10536        && ok(last)
10537        && let Some(s) = roster.iter().find(|s| s.id == last)
10538    {
10539        return s.clone();
10540    }
10541    if ok(&spec.id) {
10542        return spec;
10543    }
10544    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10545    roster
10546        .get(from + 1..)
10547        .into_iter()
10548        .flatten()
10549        .chain(roster.iter())
10550        .find(|s| ok(&s.id))
10551        .cloned()
10552        .unwrap_or(spec)
10553}
10554
10555/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10556/// seed so a CLI that mints its session id up front (`--session-id`) never
10557/// reuses the uuid the previous agent already opened under the same seat key.
10558pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10559    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10560}
10561
10562/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10563/// so the classifier below and the code that builds the message cannot drift.
10564const TIMED_OUT: &str = "timed out";
10565
10566impl FailClass {
10567    /// `None` for an answer; otherwise how the turn failed.
10568    fn of(out: &AgentOutcome) -> Option<Self> {
10569        match out {
10570            AgentOutcome::Ok(_) => None,
10571            AgentOutcome::Quota(_) => Some(Self::Quota),
10572            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10573            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10574            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10575            // Not a failure of the agent. Callers check for it first; `None`
10576            // here means "no class to hand over on".
10577            AgentOutcome::ResourceBlocked(_) => None,
10578        }
10579    }
10580
10581    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10582    fn stem_word(&self) -> &'static str {
10583        match self {
10584            Self::Quota => "quota",
10585            _ => "handover",
10586        }
10587    }
10588}
10589
10590/// The message's first line with its variable parts removed — digit runs and
10591/// path-like tokens — so "exit code 2" and "exit code 7" read
10592/// as one kind of failure.
10593fn failure_signature(msg: &str) -> String {
10594    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10595    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10596    // the historical machine key while the process diagnosis uses readable prose.
10597    if let Some(code) = line
10598        .strip_prefix("process exited with no usable output (exit code ")
10599        .and_then(|s| s.strip_suffix(')'))
10600        .and_then(|s| s.parse::<i32>().ok())
10601    {
10602        line = format!("exited with some({code}) and no usable output");
10603    } else if matches!(
10604        line.as_str(),
10605        "process exited with no usable output (no exit code reported)"
10606            | "process exited with no usable output (exit code unavailable)"
10607    ) {
10608        line = "exited with none and no usable output".to_owned();
10609    }
10610    let mut out = Vec::new();
10611    for word in line.split_whitespace() {
10612        if word.contains('/') || word.contains('\\') {
10613            out.push("<path>".to_owned());
10614            continue;
10615        }
10616        let mut w = String::new();
10617        let mut in_digits = false;
10618        for c in word.chars() {
10619            if c.is_ascii_digit() {
10620                if !in_digits {
10621                    w.push('#');
10622                }
10623                in_digits = true;
10624            } else {
10625                in_digits = false;
10626                w.push(c);
10627            }
10628        }
10629        out.push(w);
10630    }
10631    out.join(" ").chars().take(120).collect()
10632}
10633
10634/// Whether a seat that just failed with `cur` may go to the next roster agent.
10635/// A quota or a timeout always may. Any other failure may not when the agent
10636/// before it failed the same way: an error the prompt causes would otherwise
10637/// walk the whole roster. `prev` is the class of the immediately preceding
10638/// agent's failure, so a quota or timeout in between breaks the run of
10639/// identical failures by itself.
10640fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10641    match cur {
10642        FailClass::Quota | FailClass::Timeout => true,
10643        FailClass::Other(_) => prev != Some(cur),
10644    }
10645}
10646
10647/// A short human reason for a failed outcome, for the handover record.
10648fn fail_reason(out: &AgentOutcome) -> String {
10649    match out {
10650        AgentOutcome::Ok(_) => String::new(),
10651        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10652        AgentOutcome::Dropped(o) => format!(
10653            "the CLI dropped the stream ({})",
10654            o.dropped
10655                .as_ref()
10656                .map(|d| d.why.as_str())
10657                .unwrap_or("it ended without delivering its answer")
10658        ),
10659        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10660            e.lines().next().unwrap_or("").chars().take(160).collect()
10661        }
10662    }
10663}
10664
10665/// Note one handover in the run: the structured record and, in the timeline,
10666/// the sentence a person reads. A quota keeps the wording it always had.
10667pub(crate) fn record_handover(
10668    state: &mut RunState,
10669    node: &str,
10670    seat: &str,
10671    from: &str,
10672    to: &str,
10673    class: &FailClass,
10674    reason: &str,
10675) {
10676    let message = if *class == FailClass::Quota {
10677        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10678    } else {
10679        format!("{seat}: handed over {from} -> {to} ({reason})")
10680    };
10681    state.event(node, message);
10682    state.handovers.push(Handover {
10683        at: Timestamp::now(),
10684        node: node.to_owned(),
10685        seat: seat.to_owned(),
10686        from: from.to_owned(),
10687        to: to.to_owned(),
10688        reason: reason.to_owned(),
10689    });
10690}
10691
10692/// Did this reply report running a command whose own CLI never confirmed an
10693/// exit status?
10694///
10695/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10696/// command *finished* (see that type's own doc), so this can only be `true`
10697/// for a command whose completion event carried no readable exit code — not
10698/// for one that simply is not mentioned at all. That is the one signal this
10699/// crate can read, from the same record `state.jobs` renders, about a reply
10700/// standing next to work its own CLI cannot vouch for finishing; it is
10701/// deliberately not a check on the exit code's *value* (a fixer legitimately
10702/// runs a command that fails mid-iteration before it succeeds) and not a
10703/// guess at a command still running in the background (which emits no event
10704/// at all, and so leaves no evidence here to find).
10705fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10706    commands.iter().any(|c| c.exit_code.is_none())
10707}
10708
10709/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10710/// trusted as a verified no-op — the adoption guard's own text-level half.
10711///
10712/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10713/// timed out): a marker only earns the benefit of the doubt from a turn the
10714/// CLI itself vouches for finishing properly, the same house style
10715/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10716/// *fix* report to for `commands`. A candidate that timed out, exited
10717/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10718/// is, whatever prose it wrote — this returns `None` before it ever looks at
10719/// `text`. The remaining guards (the tree really is empty, the evidence is
10720/// non-empty) are the caller's: this only reads what the reply *claimed*.
10721fn verified_noop_claim(
10722    usable: bool,
10723    commands: &[agent::CommandEvidence],
10724    text: &str,
10725) -> Option<String> {
10726    (usable && !has_unconfirmed_command(commands))
10727        .then(|| verdict::verified_noop(text))
10728        .flatten()
10729}
10730
10731fn short(commit: &str) -> String {
10732    commit.chars().take(7).collect()
10733}
10734
10735fn make_executable(path: &Path) -> Result<()> {
10736    #[cfg(unix)]
10737    {
10738        use std::os::unix::fs::PermissionsExt as _;
10739        let mut perms = std::fs::metadata(path)?.permissions();
10740        perms.set_mode(0o755);
10741        std::fs::set_permissions(path, perms)?;
10742    }
10743    #[cfg(not(unix))]
10744    {
10745        let _ = path;
10746    }
10747    Ok(())
10748}
10749
10750/// What every seat in one batch shares: where the answers are attributed, the
10751/// prompt overlay they inherit, and the build cache they are told to use.
10752///
10753/// A struct rather than four more parameters: `wave` also needs the run's
10754/// state (to record who is answering right now) and the attempt number, and
10755/// eight positional arguments is both unreadable and a clippy error.
10756struct WaveCtx<'a> {
10757    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10758    /// paid for it.
10759    run: &'a str,
10760    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10761    node: &'a str,
10762    prompts: &'a Prompts,
10763    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10764    cache: Option<&'a Path>,
10765    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10766    /// `JobRecord::round`. `None` for every other node.
10767    round: Option<usize>,
10768    /// Carry each seat's failed-agent history across waves (the review loop
10769    /// only): start-of-round priority and handover choice read
10770    /// [`RunState::seat_history`], and every answer or failure writes it.
10771    carry_seats: bool,
10772    /// The run's park / interrupt requests, so a wait for a shared resource
10773    /// can be abandoned the moment the operator asks to park.
10774    pauses: Option<(&'a Pause, &'a Pause)>,
10775}
10776
10777/// Shared scheduling policy for operations and spawned seat invocations.
10778fn repository_wait(
10779    state: &RunState,
10780    pauses: Option<(&Pause, &Pause)>,
10781) -> crate::repo_lock::WaitPolicy {
10782    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10783    let inherited = crate::repo_lock::WaitPolicy::current();
10784    crate::repo_lock::WaitPolicy::new(
10785        Duration::from_secs(state.config.graph.timeout_resource_wait),
10786        move || {
10787            inherited.cancelled()
10788                || pauses
10789                    .as_ref()
10790                    .is_some_and(|(a, b)| a.parked() || b.parked())
10791        },
10792    )
10793}
10794
10795fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10796    if let Some(timeout) = error.downcast_ref::<agent::IsolatedSeatWait>() {
10797        state.event("resource", timeout.0.description());
10798        state.resource_block = Some(timeout.0.clone());
10799        return;
10800    }
10801    if error.is::<crate::repo_lock::ResourceBlocked>() {
10802        state.resource_block = Some(crate::run::ResourceBlock {
10803            cause: crate::run::ResourceBlockCause::Other,
10804            node: "repository".into(),
10805            kind: if error.is::<crate::build_target::AdmissionDiskWait>() {
10806                "disk admission"
10807            } else if error.to_string().contains("disk pressure") {
10808                "disk pressure"
10809            } else if error.to_string().contains("disk admission") {
10810                "disk admission"
10811            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10812                "incremental cache admission"
10813            } else {
10814                "repository coordination"
10815            }
10816            .into(),
10817            detail: format!("{error:#}"),
10818            at: Timestamp::now(),
10819        });
10820        state.event("resource", error.to_string());
10821    }
10822}
10823
10824async fn coordinate_operation(
10825    state: &mut RunState,
10826    cwd: &Path,
10827    wait: &crate::repo_lock::WaitPolicy,
10828) -> Result<Option<crate::repo_lock::SeatLease>> {
10829    let result =
10830        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10831            .await;
10832    if let Err(e) = &result {
10833        record_repository_block(state, e);
10834    }
10835    result
10836}
10837
10838/// Run one job, honouring the parallelism budget.
10839async fn run_one(
10840    job: SeatJob,
10841    sem: Arc<Semaphore>,
10842    ctx: &WaveCtx<'_>,
10843    state: &mut RunState,
10844    attempt: usize,
10845) -> (SeatState, AgentOutcome) {
10846    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10847        .await
10848        .pop()
10849        .expect("one job in, one result out");
10850    (seat, out)
10851}
10852
10853/// Run every job concurrently, capped by the semaphore, preserving order.
10854///
10855/// Every seat in the batch is recorded into [`RunState::active`] before the
10856/// wave starts and cleared as each answer lands, so the run's own record says
10857/// who is still being waited on rather than only who finished.
10858async fn wave(
10859    jobs: Vec<SeatJob>,
10860    sem: Arc<Semaphore>,
10861    ctx: &WaveCtx<'_>,
10862    state: &mut RunState,
10863    attempt: usize,
10864) -> Vec<(usize, SeatState, AgentOutcome)> {
10865    let WaveCtx {
10866        run,
10867        node,
10868        prompts,
10869        cache,
10870        round,
10871        carry_seats: _,
10872        pauses,
10873    } = *ctx;
10874    let cache = cache.filter(|_| !crate::build_target::isolated(&state.config));
10875    for job in &jobs {
10876        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10877    }
10878    if let Err(e) = state.save() {
10879        // A failed persist of "who is answering right now" must not abort the
10880        // wave: the seats are already being asked, and the alternative is
10881        // losing the answers to save a status line nobody may even be
10882        // watching.
10883        tracing::warn!("could not persist in-progress seats: {e:#}");
10884    }
10885    // Shared-mode waves keep their existing single lease. Isolated writers
10886    // acquire their own checkout's target at seat admission.
10887    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10888    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10889    // never borrowed from a job's agent budget (and never charged to it either,
10890    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10891    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10892    let cache_guard = if let Some(cache_dir) = cache {
10893        if jobs_had_a_writer {
10894            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10895            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10896            match acquire_cache_lease(
10897                state,
10898                cache_dir,
10899                &owner,
10900                resource_limit,
10901                node,
10902                node,
10903                &cancel,
10904            )
10905            .await
10906            {
10907                Ok(Some(g)) => Some(g),
10908                // Parking: leave without spawning anything. The run parks at
10909                // the next boundary and the seats are asked afresh on resume.
10910                Ok(None) => {
10911                    if let Some(block) = &mut state.resource_block {
10912                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10913                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10914                        };
10915                    }
10916                    return block_wave(
10917                        jobs,
10918                        state,
10919                        node,
10920                        attempt,
10921                        "parked while waiting for the shared build cache",
10922                    );
10923                }
10924                // An unreadable lease is not contention: keep the old
10925                // fallback (no shared cache for this wave's seats).
10926                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10927                Err(e) => {
10928                    // Writing seats may not build into a cache this run does
10929                    // not hold, and an agent that cannot be given one is not
10930                    // asked at all: the run stops resumably instead.
10931                    let why = format!("{e:#}");
10932                    state.event(
10933                        "resource",
10934                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10935                    );
10936                    return block_wave(jobs, state, node, attempt, &why);
10937                }
10938            }
10939        } else {
10940            None
10941        }
10942    } else {
10943        None
10944    };
10945    // Every job keeps its whole budget: the wait above was not the agent's.
10946    // Restart the seats' clocks so "started" means "being spawned now".
10947    state.seats_spawning(node, attempt);
10948    if let Err(e) = state.save() {
10949        tracing::warn!("could not persist the start of a wave: {e:#}");
10950    }
10951    // Queueing behind this wave's own seats for a parallelism slot is normal
10952    // (each of them is bounded by its own agent timeout); only a wait beyond
10953    // all of them plus the resource limit means the slots never came back.
10954    let slot_limit = resource_limit
10955        + jobs
10956            .iter()
10957            .map(|j| j.timeout)
10958            .max()
10959            .unwrap_or_default()
10960            .saturating_mul(jobs.len() as u32);
10961    let mut set = tokio::task::JoinSet::new();
10962    // Seats announce themselves here the moment they hold a parallelism
10963    // permit, i.e. just before their CLI is spawned.
10964    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10965    let overlay = prompts.overlay(node);
10966    let repository_wait = repository_wait(state, pauses);
10967    for (i, mut job) in jobs.into_iter().enumerate() {
10968        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10969        if cache.is_some() {
10970            job.prompt.push('\n');
10971            job.prompt
10972                .push_str(&prompt::build_cache_note(node, job.allow_write));
10973        }
10974        let sem = Arc::clone(&sem);
10975        let run = run.to_owned();
10976        let node = node.to_owned();
10977        // Only implementers were told about the task's attachments, so only
10978        // their seats get the directory widened for reading.
10979        let attachments = if node == "implement" {
10980            state.attachments.clone()
10981        } else {
10982            Vec::new()
10983        };
10984        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10985        // `prompt::build_cache_note`'s doc for why setting it anyway is
10986        // exactly how a sandboxed reviewer's write refusal got reported as a
10987        // defect in the patch, not a property of its own seat. And a
10988        // write-allowed one is handed it only when the lease above was
10989        // actually acquired: a wave that could not get it (`cache_guard` is
10990        // `None`, see its own comment) must not send seats to build into a
10991        // directory this run does not hold - that is the exact concurrent,
10992        // unmanaged-write race this module exists to prevent, not something
10993        // "proceeding anyway" is allowed to reintroduce.
10994        let cache = cache
10995            .filter(|_| job.allow_write && cache_guard.is_some())
10996            .map(Path::to_path_buf);
10997        let spawned_tx = spawned_tx.clone();
10998        let repository_wait = repository_wait.clone();
10999        let coordination = state.config.coordination.clone();
11000        let fetched_base = state.fetched_base.clone();
11001        set.spawn(async move {
11002            // A wait for a parallelism slot is scheduling, not the agent: it
11003            // is bounded by its own limit and ends as a resource block, never
11004            // as a timeout of the agent that was not started.
11005            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
11006                Ok(p) => p,
11007                Err(_) => {
11008                    return (
11009                        i,
11010                        job.seat,
11011                        AgentOutcome::ResourceBlocked(format!(
11012                            "no parallel slot became free within {}s",
11013                            slot_limit.as_secs()
11014                        )),
11015                        None,
11016                        false,
11017                    );
11018                }
11019            };
11020            spawned_tx.send(job.seat.key.clone()).ok();
11021            let mut seat = job.seat;
11022            let out = repository_wait
11023                .scope(agent::invoke(
11024                    &job.spec,
11025                    &mut seat,
11026                    &Invocation {
11027                        coordination: coordination.as_ref(),
11028                        fetched_base: fetched_base.as_ref(),
11029                        cwd: &job.cwd,
11030                        prompt: &job.prompt,
11031                        timeout: job.timeout,
11032                        allow_write: job.allow_write,
11033                        sessions: job.sessions,
11034                        artifacts: &job.artifacts,
11035                        stem: &job.stem,
11036                        run: &run,
11037                        node: &node,
11038                        cache_dir: cache.as_deref(),
11039                        attachments: &attachments,
11040                        writable: &[],
11041                    },
11042                ))
11043                .await;
11044            let cache_block = out
11045                .as_ref()
11046                .err()
11047                .and_then(|error| error.downcast_ref::<agent::IsolatedSeatWait>())
11048                .map(|timeout| timeout.0.clone());
11049            let interrupted = out
11050                .as_ref()
11051                .err()
11052                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
11053            let out = match out {
11054                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
11055                Ok(o) if o.usable() => AgentOutcome::Ok(o),
11056                // Billed work the CLI failed to hand over is not an ordinary
11057                // failure, but its text is the CLI's raw error JSON, not an
11058                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
11059                // read it as one by forgetting to check. `usable()` is always
11060                // false here (dropped implies an empty response), so this has
11061                // to be checked before the catch-all `Failed` below or the
11062                // one shape this exists for is lost with the rest.
11063                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
11064                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
11065                Ok(o) => AgentOutcome::Failed(format!(
11066                    "process exited with no usable output ({})",
11067                    crate::notices::exit_status(o.exit_code)
11068                )),
11069                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
11070                    AgentOutcome::ResourceBlocked(e.to_string())
11071                }
11072                Err(e) => AgentOutcome::Failed(e.to_string()),
11073            };
11074            (i, seat, out, cache_block, interrupted)
11075        });
11076    }
11077    drop(spawned_tx);
11078    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
11079    loop {
11080        let joined = tokio::select! {
11081            Some(key) = spawned_rx.recv() => {
11082                // The clock of this seat starts now, not when it queued.
11083                state.seat_spawned(&key);
11084                if let Err(e) = state.save() {
11085                    tracing::warn!("could not persist a seat's start: {e:#}");
11086                }
11087                continue;
11088            }
11089            joined = set.join_next() => joined,
11090        };
11091        let Some(joined) = joined else {
11092            break;
11093        };
11094        let (i, seat, out, cache_block, interrupted) = match joined {
11095            Ok(v) => v,
11096            // No seat to clear: a panicked task never reported which one it
11097            // was. The defensive sweep below this loop is what stops that
11098            // seat's `active` entry from surviving forever.
11099            Err(e) => {
11100                tracing::error!("agent task panicked: {e}");
11101                continue;
11102            }
11103        };
11104        state.seat_finished(&seat.key);
11105        if interrupted
11106            && node == "implement"
11107            && let Some(label) = state
11108                .candidates
11109                .iter()
11110                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
11111                .map(|candidate| candidate.label)
11112        {
11113            state.candidate_preservation.insert(
11114                label,
11115                "interrupted implementation awaiting persistence".into(),
11116            );
11117            state.interrupted_implementations.insert(label);
11118        }
11119        if let Some(block) = cache_block
11120            && state.resource_block.is_none()
11121        {
11122            state.event("resource", block.description());
11123            state.resource_block = Some(block);
11124        }
11125        if let AgentOutcome::ResourceBlocked(why) = &out
11126            && state.resource_block.is_none()
11127        {
11128            state.resource_block = Some(crate::run::ResourceBlock {
11129                cause: crate::run::ResourceBlockCause::Other,
11130                node: node.to_owned(),
11131                kind: if why.contains("disk pressure") {
11132                    "disk pressure"
11133                } else if why.contains("disk admission") {
11134                    "disk admission"
11135                } else if why.contains(crate::incremental::BLOCKED) {
11136                    "incremental cache admission"
11137                } else if why.contains(crate::build_target::KIND) {
11138                    crate::build_target::KIND
11139                } else if why.contains("repository coordination") {
11140                    "repository coordination"
11141                } else {
11142                    "parallel slot"
11143                }
11144                .to_owned(),
11145                detail: why.clone(),
11146                at: Timestamp::now(),
11147            });
11148            let activity = if interrupted {
11149                "agent execution interrupted"
11150            } else {
11151                "no agent was started"
11152            };
11153            state.event("resource", format!("{node}: {activity} - {why}"));
11154        }
11155        record_jobs(state, node, round, &seat.key, &out);
11156        if let Err(e) = state.save() {
11157            tracing::warn!("could not persist a seat's completion: {e:#}");
11158        }
11159        if collected.len() <= i {
11160            collected.resize_with(i + 1, || None);
11161        }
11162        collected[i] = Some((i, seat, out));
11163    }
11164    // Belt-and-braces for the panic branch above: every seat this exact batch
11165    // started shares this `(node, attempt)` pair, and every seat that finished
11166    // normally already cleared itself, so anything left tagged with it here
11167    // can only be a panicked task's leftover. Cleared unconditionally rather
11168    // than left to read as still answering forever.
11169    if state
11170        .active
11171        .values()
11172        .any(|a| a.node == node && a.attempt == attempt)
11173    {
11174        state
11175            .active
11176            .retain(|_, a| !(a.node == node && a.attempt == attempt));
11177        if let Err(e) = state.save() {
11178            tracing::warn!("could not persist the end of a wave: {e:#}");
11179        }
11180    }
11181    // Whether or not the lease above was actually held, several worktrees
11182    // may just have built into the cache with nothing here able to name one
11183    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
11184    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
11185    // clean it might not have strictly needed; trusting a stale match would
11186    // cost it a wrong answer.
11187    if let Some(cache_dir) = cache
11188        && jobs_had_a_writer
11189    {
11190        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
11191    }
11192    if let Some(guard) = cache_guard {
11193        guard.release();
11194    }
11195    collected.into_iter().flatten().collect()
11196}
11197
11198/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
11199/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
11200/// this is data collection, not the fix-specific completion contract in
11201/// [`Runner::continue_fix_report`], and applies regardless of which node
11202/// asked.
11203///
11204/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11205/// evidence from; `Failed` does not, and correctly contributes nothing — a
11206/// timeout or crash is not itself evidence about a command the seat may have
11207/// started.
11208fn record_jobs(
11209    state: &mut RunState,
11210    node: &str,
11211    round: Option<usize>,
11212    seat: &str,
11213    out: &AgentOutcome,
11214) {
11215    let commands: &[agent::CommandEvidence] = match out {
11216        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11217        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11218    };
11219    let checked_at = Timestamp::now();
11220    for c in commands {
11221        state.jobs.push(JobRecord {
11222            node: node.to_owned(),
11223            round,
11224            seat: seat.to_owned(),
11225            id: c.id.clone(),
11226            description: c.description.clone(),
11227            checked_at,
11228            status: match c.exit_code {
11229                Some(0) => JobStatus::Completed,
11230                Some(_) => JobStatus::Failed,
11231                None => JobStatus::Unknown,
11232            },
11233            exit_code: c.exit_code,
11234            result_summary: c.result_summary.clone(),
11235            source: c.source.clone(),
11236        });
11237    }
11238}
11239
11240/// Is a review round clean, given how many reviewer seats answered against
11241/// how many the round expected?
11242///
11243/// A seat that never answered (timeout, crash, unparsable output) is not a
11244/// seat that read the patch and found nothing — treating it as such is
11245/// exactly the bug this function exists to close. Under the default `block`
11246/// policy a missing seat can never be clean; `warn` still requires the seats
11247/// that *did* answer to have found nothing blocking and verification to be
11248/// green.
11249///
11250/// `quota_missing` narrows that `block` default for exactly one cause of
11251/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11252/// a session limit lifts by the very next round buys nothing — the seat is
11253/// asked again with the same quota — so once every missing seat is accounted
11254/// for by a quota loss (and at least one seat *did* answer, so a decision has
11255/// something to rest on) the round is decided on the panel that could answer,
11256/// same as `warn` would. A panel that lost every seat to quota is not
11257/// decided here: `answered == 0` falls through to the existing `block`
11258/// fallback so a fully collapsed panel still waits rather than landing on no
11259/// review at all.
11260fn round_is_clean(
11261    blocking: usize,
11262    e2e_ok: bool,
11263    answered: usize,
11264    expected: usize,
11265    quota_missing: usize,
11266    policy: IncompleteReviewPolicy,
11267) -> bool {
11268    if blocking != 0 || !e2e_ok {
11269        return false;
11270    }
11271    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11272        return true;
11273    }
11274    answered > 0 && expected - answered <= quota_missing
11275}
11276
11277/// The review loop's own conclusion, derived entirely from its persisted
11278/// round records and the round budget that produced them — never from
11279/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11280/// recomputes the identical answer regardless of what an earlier node in the
11281/// same walk, or a previous walk, did to `status`.
11282///
11283/// `None` while more rounds remain to try, including when review never ran
11284/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11285/// gone clean, or the budget is spent, or the tree has stopped moving (see
11286/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11287///
11288/// - An incomplete panel that raised nothing is missing input, not a
11289///   verified tree — never a hand-off candidate, whatever verification said
11290///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11291/// - Otherwise, green e2e on the last round hands off (see
11292///   [`Runner::stop_reviewing`]); red e2e blocks.
11293///
11294/// A last round whose own verification is still `ResourceBlocked` — magi
11295/// itself never got a command to run, not evidence the patch is broken —
11296/// is neither: this returns `None` for it too, the same as "more rounds
11297/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11298/// handling of that shape) instead of this cheap recomputation guessing a
11299/// verdict a real attempt never produced.
11300pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11301    if max_rounds == 0
11302        || reviews.last().is_some_and(|r| {
11303            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11304        })
11305    {
11306        return Some(RunStatus::Gating);
11307    }
11308    let last = reviews.last()?;
11309    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11310    if reviews.len() < max_rounds && !stagnant {
11311        return None;
11312    }
11313    if last.incomplete() && last.blocking == 0 {
11314        return Some(RunStatus::Blocked);
11315    }
11316    if last.e2e_status() == E2eStatus::ResourceBlocked
11317        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11318    {
11319        return None;
11320    }
11321    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11322        RunStatus::Gating
11323    } else {
11324        RunStatus::Blocked
11325    })
11326}
11327
11328/// How long a re-ask may take, given the budget the first attempt had.
11329///
11330/// A `nudged` retry is a request to restate an answer the seat has already
11331/// worked out: it carries no new work, so it does not deserve the original
11332/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11333/// 133 seconds while a third sat for over ten minutes on a resumed session
11334/// holding 410 KB of prior output - and because the retry had inherited the
11335/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11336/// judging round whose other seats were long finished.
11337///
11338/// A quarter of the budget, with a floor so that a deliberately short timeout
11339/// does not collapse to nothing. A retry that re-sends the whole prompt
11340/// (because the seat kept no context) is the original job again, and keeps the
11341/// original budget.
11342pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11343    if nudged {
11344        (full / 4).max(Duration::from_secs(120)).min(full)
11345    } else {
11346        full
11347    }
11348}
11349
11350/// Run a wave and parse each reply, re-asking the seats whose reply was
11351/// unusable.
11352///
11353/// The re-ask is a nudge rather than the whole prompt again when the seat still
11354/// holds its conversation, which is the difference between a cheap retry and
11355/// paying for the entire candidate set twice.
11356///
11357/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11358/// successor in `roster` is handed to it instead of being re-asked: the
11359/// handover is the retry. A seat with no successor left (a single-agent
11360/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11361/// the asks to one seat in one node number at most
11362/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11363/// is asked once (a dropped stream is nudged first), and only the last agent
11364/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11365/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11366/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11367/// other failure stops the chain when the previous agent failed the same way
11368/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11369/// [`has_context`] is false and the job's own full prompt and full budget are
11370/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11371/// intermediate ones are not losses) and is returned as a failure like any
11372/// other absent seat — the caller decides whether the panel still has a
11373/// quorum. An empty `roster` disables handover: failures are nudged as they
11374/// always were, and a quota is simply lost. A reply that fails to parse or
11375/// validate is the prompt's doing and is only ever nudged, never handed over.
11376///
11377/// The returned [`SeatState`] names the agent that answered (or tried last).
11378#[allow(clippy::too_many_arguments)]
11379async fn ask_json_wave<T>(
11380    jobs: Vec<SeatJob>,
11381    sem: Arc<Semaphore>,
11382    retries: usize,
11383    roster: &[AgentSpec],
11384    ctx: &WaveCtx<'_>,
11385    losses: &mut Vec<QuotaLoss>,
11386    state: &mut RunState,
11387    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11388) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11389where
11390    T: serde::de::DeserializeOwned + Send + 'static,
11391{
11392    ask_wave_with(
11393        jobs,
11394        sem,
11395        retries,
11396        roster,
11397        ctx,
11398        losses,
11399        state,
11400        &|text: &str| {
11401            let v = verdict::extract_json::<T>(text)?;
11402            validate(&v)?;
11403            Ok(v)
11404        },
11405    )
11406    .await
11407}
11408
11409/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11410/// a node whose answer is prose (deliberation) shares the same handover,
11411/// failure classification, quota bookkeeping and bounds instead of a copy.
11412///
11413/// A seat handed to another roster agent is sent the job's `handover` prompt
11414/// (when it has one) rather than `prompt`: the new agent has no session, so a
11415/// resume-style prompt would be incomplete. That holds for the handover ask
11416/// and for every nudge to that agent whose `has_context` is false.
11417#[allow(clippy::too_many_arguments)]
11418async fn ask_wave_with<T>(
11419    jobs: Vec<SeatJob>,
11420    sem: Arc<Semaphore>,
11421    retries: usize,
11422    roster: &[AgentSpec],
11423    ctx: &WaveCtx<'_>,
11424    losses: &mut Vec<QuotaLoss>,
11425    state: &mut RunState,
11426    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11427) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11428where
11429    T: Send + 'static,
11430{
11431    let n = jobs.len();
11432    let originals: Vec<SeatJob> = jobs;
11433    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11434    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11435    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11436    // answer, N once it has gone through N nudges. Read back once this
11437    // returns, so a caller building a history record (`ReviewRecord`) can
11438    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11439    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11440    // field's own doc.
11441    let mut nudges: Vec<usize> = vec![0; n];
11442    // The agent now occupying each seat, the ids it has already been through,
11443    // where in the roster the walk began, the class of the last failure, and
11444    // the stem word of a handover not yet asked (full prompt, full budget).
11445    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11446    let mut tried: Vec<BTreeSet<String>> = specs
11447        .iter()
11448        .map(|s| BTreeSet::from([s.id.clone()]))
11449        .collect();
11450    let starts: Vec<usize> = specs
11451        .iter()
11452        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11453        .collect();
11454    let carry = ctx.carry_seats && !roster.is_empty();
11455    // Carried across rounds: ids that failed the seat earlier, and how the
11456    // last failure went (so a repeat of it is not handed over again).
11457    let carried: Vec<BTreeSet<String>> = originals
11458        .iter()
11459        .map(|j| {
11460            state
11461                .seat_history
11462                .get(&j.seat.key)
11463                .filter(|_| carry)
11464                .map(|h| h.failed.clone())
11465                .unwrap_or_default()
11466        })
11467        .collect();
11468    let mut prev: Vec<Option<FailClass>> = originals
11469        .iter()
11470        .map(|j| {
11471            state
11472                .seat_history
11473                .get(&j.seat.key)
11474                .filter(|_| carry)
11475                .and_then(|h| h.last_fail.clone())
11476        })
11477        .collect();
11478    let next_agent =
11479        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11480            let others: BTreeSet<String> = specs
11481                .iter()
11482                .enumerate()
11483                .filter(|(j, _)| *j != i)
11484                .map(|(_, s)| s.id.clone())
11485                .collect();
11486            pick_successor(
11487                roster,
11488                starts[i],
11489                tried,
11490                carry.then(|| &carried[i]),
11491                &others,
11492            )
11493            .cloned()
11494        };
11495    let mut fresh: Vec<Option<String>> = vec![None; n];
11496    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11497    let mut pending: Vec<usize> = (0..n).collect();
11498
11499    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11500    // asks: an agent with a successor is asked once and handed over, and only
11501    // a seat with no successor spends `retries` nudges on the same agent. This
11502    // only guarantees the loop's own termination whatever those say.
11503    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11504    for round in 0..max_rounds {
11505        if pending.is_empty() {
11506            break;
11507        }
11508        let mut batch = Vec::with_capacity(pending.len());
11509        let mut renudged: Vec<&str> = Vec::new();
11510        for &i in &pending {
11511            let src = &originals[i];
11512            // A seat now held by another agent than the job named has no
11513            // session of its own: it gets the full-context prompt whenever
11514            // it is asked in full (the handover ask, a nudge it cannot
11515            // resume).
11516            let full: &str = match &src.handover {
11517                Some(h) if specs[i].id != src.spec.id => h,
11518                _ => &src.prompt,
11519            };
11520            // The prompt and the budget are one decision: a nudge restates
11521            // finished work, a re-sent prompt redoes it.
11522            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11523                (
11524                    full.to_owned(),
11525                    src.timeout,
11526                    format!("{}-{word}-{}", src.stem, specs[i].id),
11527                )
11528            } else if nudges[i] == 0 {
11529                (full.to_owned(), src.timeout, src.stem.clone())
11530            } else {
11531                renudged.push(src.seat.key.as_str());
11532                let why = done[i]
11533                    .as_ref()
11534                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11535                    .unwrap_or_else(|| "no parsable answer".to_owned());
11536                let nudge = prompt::nudge(&why);
11537                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11538                let prompt = if nudged {
11539                    nudge
11540                } else {
11541                    format!("{full}\n\n---\n\n{nudge}")
11542                };
11543                (
11544                    prompt,
11545                    retry_budget(src.timeout, nudged),
11546                    format!("{}-retry{}", src.stem, nudges[i]),
11547                )
11548            };
11549            batch.push(SeatJob {
11550                spec: specs[i].clone(),
11551                seat: seats[i].clone(),
11552                cwd: src.cwd.clone(),
11553                prompt,
11554                timeout,
11555                allow_write: src.allow_write,
11556                sessions: src.sessions,
11557                artifacts: src.artifacts.clone(),
11558                stem,
11559                handover: None,
11560            });
11561        }
11562
11563        if !renudged.is_empty() {
11564            state.event(
11565                ctx.node,
11566                format!("retry {round}: re-asking {}", renudged.join(", ")),
11567            );
11568        }
11569        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11570        let mut still = Vec::new();
11571        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11572            seats[i] = seat;
11573            // No agent was asked: nothing to hand over, nudge, remember or
11574            // charge. The caller sees `resource_block` and stops the node.
11575            if let AgentOutcome::ResourceBlocked(why) = &out {
11576                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11577                continue;
11578            }
11579            let class = FailClass::of(&out);
11580            // A dropped stream is nudged first (the conversation is still
11581            // there to pick up); only a seat whose nudges are spent hands over.
11582            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11583                && nudges[i] < retries
11584                && !roster.is_empty();
11585            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11586                let next = should_hand_over(prev[i].as_ref(), &cur)
11587                    .then(|| next_agent(i, &tried[i], &specs))
11588                    .flatten();
11589                if carry {
11590                    let h = state
11591                        .seat_history
11592                        .entry(originals[i].seat.key.clone())
11593                        .or_default();
11594                    h.failed.insert(specs[i].id.clone());
11595                    h.last_fail = Some(cur.clone());
11596                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11597                        h.last_ok = None;
11598                    }
11599                    // Saved before the next agent is asked, so a restart in
11600                    // between does not forget who failed.
11601                    if let Err(e) = state.save() {
11602                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11603                    }
11604                }
11605                if let Some(next) = next {
11606                    record_handover(
11607                        state,
11608                        ctx.node,
11609                        &originals[i].seat.key,
11610                        &specs[i].id,
11611                        &next.id,
11612                        &cur,
11613                        &fail_reason(&out),
11614                    );
11615                    tried[i].insert(next.id.clone());
11616                    prev[i] = Some(cur.clone());
11617                    seats[i] =
11618                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11619                    specs[i] = next;
11620                    fresh[i] = Some(cur.stem_word().to_owned());
11621                    nudges[i] = 0;
11622                    done[i] = Some(Err(anyhow::anyhow!(
11623                        "handed over after: {}",
11624                        fail_reason(&out)
11625                    )));
11626                    still.push(i);
11627                    continue;
11628                }
11629            }
11630            if carry
11631                && !nudge_first
11632                && let Some(cur) = class.clone()
11633            {
11634                // The chain ended here (no successor, or a repeated failure).
11635                let h = state
11636                    .seat_history
11637                    .entry(originals[i].seat.key.clone())
11638                    .or_default();
11639                h.failed.insert(specs[i].id.clone());
11640                h.last_fail = Some(cur);
11641            }
11642            let parsed = match out {
11643                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11644                AgentOutcome::Quota(o) => {
11645                    last_quota[i] = o.quota.clone();
11646                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11647                }
11648                // Not a parseable answer: the nudge loop re-asks it, which is
11649                // exactly what a dropped stream needs. Just don't hand its raw
11650                // error JSON to `extract_json`.
11651                AgentOutcome::Dropped(o) => {
11652                    let why = o
11653                        .dropped
11654                        .as_ref()
11655                        .map(|d| d.why.as_str())
11656                        .unwrap_or("the CLI ended the stream without delivering its answer");
11657                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11658                }
11659                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11660                    Err(anyhow::anyhow!(e))
11661                }
11662            };
11663            let quota = class == Some(FailClass::Quota);
11664            let failed = parsed.is_err();
11665            if !failed && matches!(ctx.node, "judge" | "vote") {
11666                // A validated answer restores this seat's quorum presence,
11667                // including on the normal path after an all-seat block.
11668                state
11669                    .quota
11670                    .retain(|loss| loss.seat != originals[i].seat.key);
11671            }
11672            done[i] = Some(parsed);
11673            if carry && !failed {
11674                let h = state
11675                    .seat_history
11676                    .entry(originals[i].seat.key.clone())
11677                    .or_default();
11678                h.failed.remove(&specs[i].id);
11679                h.last_ok = Some(specs[i].id.clone());
11680                h.last_fail = None;
11681            }
11682            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11683            // fail the same way; and never re-ask a seat that already parsed.
11684            // A failed agent that still has a successor is not re-asked
11685            // either: the handover was its remedy and has just been refused
11686            // (the chain stops on a repeated failure class). A seat with no
11687            // successor left (a single-agent roster, the roster's tail, or an
11688            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11689            let agent_failure = class.is_some()
11690                && !nudge_first
11691                && !roster.is_empty()
11692                && next_agent(i, &tried[i], &specs).is_some();
11693            if failed && !quota && !agent_failure && nudges[i] < retries {
11694                nudges[i] += 1;
11695                still.push(i);
11696            }
11697        }
11698        pending = still;
11699    }
11700
11701    // One loss per seat whose chain ended on a quota: the intermediate ones
11702    // were absorbed by a handover and are not losses.
11703    for (i, q) in last_quota.into_iter().enumerate() {
11704        if let Some(quota) = q {
11705            losses.push(QuotaLoss::from_report(
11706                originals[i].seat.key.clone(),
11707                ctx.node,
11708                Some(&quota),
11709            ));
11710        }
11711    }
11712
11713    // A stalled quorum already has its own recovery checkpoint. Refresh
11714    // its missing seats before deciding whether it can resume.
11715    if state.status != RunStatus::Stalled
11716        && ctx.node != "deliberate"
11717        && n > 0
11718        && losses.len() == n
11719        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11720    {
11721        block_machine_outage(state, ctx.node, losses);
11722        state.quota.append(losses);
11723    }
11724
11725    seats
11726        .into_iter()
11727        .zip(done)
11728        .zip(nudges)
11729        .map(|((seat, res), attempts)| {
11730            (
11731                seat,
11732                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11733                attempts,
11734            )
11735        })
11736        .collect()
11737}
11738
11739/// A node with no available agent has no task verdict. Preserve the node
11740/// checkpoint and record why a subsequent explicit resume is needed.
11741pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11742    let mut auth: Vec<_> = losses
11743        .iter()
11744        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11745        .map(|loss| loss.agent.clone())
11746        .collect();
11747    auth.sort();
11748    auth.dedup();
11749    let (kind, detail) = if auth.is_empty() {
11750        (
11751            "agent usage limit",
11752            "All seats reached their agent usage limits; resume after usage is available"
11753                .to_owned(),
11754        )
11755    } else {
11756        (
11757            "agent authentication",
11758            format!(
11759                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11760                auth.join(", ")
11761            ),
11762        )
11763    };
11764    state.resource_block = Some(crate::run::ResourceBlock {
11765        node: node.to_owned(),
11766        kind: kind.to_owned(),
11767        cause: crate::run::ResourceBlockCause::Other,
11768        detail: detail.clone(),
11769        at: Timestamp::now(),
11770    });
11771    state.event(node, detail);
11772}
11773
11774/// Did the wave end without asking any agent (see [`block_wave`])?
11775fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11776    results
11777        .iter()
11778        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11779}
11780
11781/// Settle a wave that never spawned anything: every job answers
11782/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11783/// else about the seat (session, history, quota) is touched.
11784fn block_wave(
11785    jobs: Vec<SeatJob>,
11786    state: &mut RunState,
11787    node: &str,
11788    attempt: usize,
11789    why: &str,
11790) -> Vec<(usize, SeatState, AgentOutcome)> {
11791    state.resource_wait = None;
11792    state
11793        .active
11794        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11795    if let Err(e) = state.save() {
11796        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11797    }
11798    jobs.into_iter()
11799        .enumerate()
11800        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11801        .collect()
11802}
11803
11804/// Acquire the shared build cache's lease, waiting out contention within
11805/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11806/// unbounded wait is never acceptable).
11807///
11808/// A first, non-blocking check happens before ever waiting; if it finds the
11809/// lease busy, that fact is logged as a `verify` event *and* flushed with
11810/// [`RunState::save`] immediately — not only once the wait finally succeeds
11811/// or gives up — so a `magi show` run by a different process while this one
11812/// is still waiting reads a `run.json` that says so, rather than whatever it
11813/// looked like before the wait started. The same applies to the terminal
11814/// failure: logged and saved before this returns `Err`, so a caller that
11815/// could not get the lease at all still leaves a legible record of why.
11816async fn acquire_cache_lease(
11817    state: &mut RunState,
11818    cache_dir: &Path,
11819    owner: &crate::cache::Owner,
11820    budget: Duration,
11821    node: &str,
11822    context: &str,
11823    cancel: &(dyn Fn() -> bool + Sync),
11824) -> Result<Option<crate::cache::Guard>> {
11825    let home = crate::run::home();
11826    let kind = if crate::build_target::isolated(&state.config) {
11827        crate::build_target::KIND
11828    } else {
11829        "shared build cache"
11830    };
11831    let started = Instant::now();
11832    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11833        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11834        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11835        Err(e) => {
11836            state.event(
11837                "verify",
11838                format!(
11839                    "{context}: could not check the {kind} at {}: {e:#}",
11840                    cache_dir.display()
11841                ),
11842            );
11843            if let Err(e2) = state.save() {
11844                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11845            }
11846            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11847        }
11848    };
11849    state.event(
11850        "verify",
11851        format!(
11852            "{context}: waiting for the {kind} at {} ({})",
11853            cache_dir.display(),
11854            busy.describe()
11855        ),
11856    );
11857    // Seats of this node are waiting, not running: say so, so the countdown a
11858    // reader sees only ever measures an agent that has been spawned.
11859    state.resource_wait = Some(crate::run::ResourceWait {
11860        node: context.to_owned(),
11861        kind: kind.to_owned(),
11862        holder: format!("{}: {}", cache_dir.display(), busy.describe()),
11863        since: Timestamp::now(),
11864        limit_secs: budget.as_secs(),
11865    });
11866    if let Err(e) = state.save() {
11867        tracing::warn!("could not persist a cache wait: {e:#}");
11868    }
11869    let remaining = budget.saturating_sub(started.elapsed());
11870    let waited = crate::cache::wait_for_unless(
11871        &home,
11872        cache_dir,
11873        owner,
11874        remaining,
11875        Duration::from_secs(5),
11876        cancel,
11877    )
11878    .await;
11879    state.resource_wait = None;
11880    let cause = match &waited {
11881        Ok(Some(_)) => None,
11882        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11883        Err(error)
11884            if error.is::<crate::cache::WaitTimeout>() && kind == crate::build_target::KIND =>
11885        {
11886            Some(crate::run::ResourceBlockCause::IsolatedCacheTimeout {
11887                target: cache_dir.to_owned(),
11888                waited_secs: started.elapsed().as_secs(),
11889                limit_secs: budget.as_secs(),
11890            })
11891        }
11892        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11893            Some(crate::run::ResourceBlockCause::Timeout {
11894                waited_secs: started.elapsed().as_secs(),
11895                limit_secs: budget.as_secs(),
11896            })
11897        }
11898        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11899    };
11900    if let Some(cause) = cause {
11901        let detail = match &waited {
11902            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11903            Err(error) => format!("{error:#}"),
11904        };
11905        state.resource_block = Some(crate::run::ResourceBlock {
11906            cause,
11907            node: node.to_owned(),
11908            kind: kind.into(),
11909            detail: format!("{}: {detail}", cache_dir.display()),
11910            at: Timestamp::now(),
11911        });
11912        state.event(
11913            "resource",
11914            state.resource_block.as_ref().unwrap().description(),
11915        );
11916    }
11917    if let Err(error) = state.save() {
11918        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11919    }
11920    waited
11921}
11922
11923/// The lease state could not be read at all (unreadable lease file, a
11924/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11925/// known to be contended, so a wave degrades to running without a shared cache
11926/// instead of stopping on a hold that [`crate::triage`] could never release.
11927#[derive(Debug)]
11928struct CacheCheckError(String);
11929
11930impl std::fmt::Display for CacheCheckError {
11931    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11932        f.write_str(&self.0)
11933    }
11934}
11935
11936impl std::error::Error for CacheCheckError {}
11937
11938/// Keep a specific acquisition outcome when the caller adds command evidence.
11939fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11940    let kind = if crate::build_target::isolated(&state.config) {
11941        crate::build_target::KIND
11942    } else {
11943        "shared build cache"
11944    };
11945    state
11946        .resource_block
11947        .get_or_insert_with(|| crate::run::ResourceBlock {
11948            cause: crate::run::ResourceBlockCause::Other,
11949            node: node.to_owned(),
11950            kind: kind.into(),
11951            detail: format!("{error:#}"),
11952            at: Timestamp::now(),
11953        });
11954    if let Err(error) = state.save() {
11955        tracing::warn!("could not persist cache resource block: {error:#}");
11956    }
11957}
11958
11959tokio::task_local! {
11960    static VERIFY_TARGET: PathBuf;
11961}
11962
11963/// Run `body` — a verify command batch — while holding the shared build
11964/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11965/// never interleave with another borrower's build against the same
11966/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11967/// timeout, or a human's own `magi review`. See the `cache` module doc for
11968/// why this matters more than Cargo's own per-target locking covers — two
11969/// *different* worktrees building the same package name/version into one
11970/// cache directory is a staleness bug, not a lock contention one.
11971///
11972/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11973/// agent waves do. Repository scheduling also has its own bounded,
11974/// cancellable resource-wait policy. Contention must not consume the
11975/// verification budget: `body` receives the full `budget` after both leases
11976/// are acquired and the cache is fresh. Without a shared cache, only
11977/// repository coordination is acquired before handing over the full budget.
11978///
11979/// A lease that cannot be acquired within the resource wait is reported as a single
11980/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11981/// verification — the same shape a spawn failure already takes in
11982/// [`run_commands`], so a caller need not special-case it.
11983#[allow(clippy::too_many_arguments)]
11984async fn with_cache_lease<'s, F, Fut>(
11985    state: &'s mut RunState,
11986    cache_dir: Option<&Path>,
11987    node: &str,
11988    seat: &str,
11989    worktree: &Path,
11990    head: &str,
11991    budget: Duration,
11992    context: &str,
11993    wait: &crate::repo_lock::WaitPolicy,
11994    body: F,
11995) -> Result<(Vec<CommandOutcome>, bool)>
11996where
11997    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11998    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11999{
12000    // Match editing waves: cache first, repository second. In particular,
12001    // pre_gate must never hold the repository while waiting for a wave's cache.
12002    let started = Instant::now();
12003    let home = crate::run::home();
12004    let isolated = crate::build_target::isolated(&state.config);
12005    let planned_target = if isolated {
12006        let plan = (|| -> Result<PathBuf> {
12007            crate::build_target::supports(&state.config)?;
12008            crate::build_target::resolve(&home, worktree)
12009        })();
12010        match plan {
12011            Ok(path) => Some(path),
12012            Err(error) => {
12013                record_cache_error(state, node, &error);
12014                return Ok((
12015                    vec![CommandOutcome {
12016                        command: "(planning isolated build cache)".into(),
12017                        code: None,
12018                        output_tail: format!("{error:#}"),
12019                        duration_ms: 0,
12020                        resource_blocked: true,
12021                    }],
12022                    false,
12023                ));
12024            }
12025        }
12026    } else {
12027        None
12028    };
12029    let cache_dir = planned_target.as_deref().or(cache_dir);
12030    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
12031    let guard = if let Some(cache_dir) = cache_dir {
12032        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
12033        match acquire_cache_lease(
12034            state,
12035            cache_dir,
12036            &owner,
12037            resource_limit,
12038            node,
12039            context,
12040            &|| wait.cancelled(),
12041        )
12042        .await
12043        .and_then(|g| {
12044            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
12045        }) {
12046            Ok(g) => Some(g),
12047            Err(e) => {
12048                record_cache_error(state, node, &e);
12049                return Ok((
12050                    vec![CommandOutcome {
12051                        command: if isolated {
12052                            "(waiting for the isolated build cache)"
12053                        } else {
12054                            "(waiting for the shared build cache)"
12055                        }
12056                        .into(),
12057                        code: None,
12058                        output_tail: e.to_string(),
12059                        duration_ms: started.elapsed().as_millis() as u64,
12060                        resource_blocked: true,
12061                    }],
12062                    false,
12063                ));
12064            }
12065        }
12066    } else {
12067        None
12068    };
12069    let _coordination = coordinate_operation(state, worktree, wait).await?;
12070    if let Some(path) = &planned_target {
12071        state.event(node, format!("isolated build target: {}", path.display()));
12072    }
12073    if let Some(cache_dir) = cache_dir.filter(|_| !isolated) {
12074        let identity = crate::cache::Identity::new(worktree, head);
12075        let fresh = crate::cache::ensure_fresh(
12076            &home,
12077            cache_dir,
12078            &identity,
12079            &state.config.verify.cargo_manifests(),
12080        );
12081        if let Err(e) = fresh {
12082            record_cache_error(state, node, &e);
12083            state.event(
12084                "verify",
12085                format!(
12086                    "{context}: could not confirm the build cache matches {} at {}: {e:#}",
12087                    worktree.display(),
12088                    short(head)
12089                ),
12090            );
12091            if let Some(guard) = guard {
12092                guard.release();
12093            }
12094            return Ok((
12095                vec![CommandOutcome {
12096                    command: "(confirming the build cache is fresh)".into(),
12097                    code: None,
12098                    output_tail: e.to_string(),
12099                    duration_ms: started.elapsed().as_millis() as u64,
12100                    resource_blocked: true,
12101                }],
12102                false,
12103            ));
12104        }
12105    }
12106    let future = body(state, budget);
12107    let (outcomes, retried, timed_out_pids) = if let Some(target) = planned_target {
12108        VERIFY_TARGET.scope(target, future).await
12109    } else {
12110        future.await
12111    };
12112    if !timed_out_pids.is_empty() {
12113        wait_for_timed_out_children_to_die(&timed_out_pids).await;
12114    }
12115    // Drop repository coordination before releasing the cache, reversing the
12116    // acquisition order. A cancelled/error future also drops both guards.
12117    drop(_coordination);
12118    if let Some(guard) = guard {
12119        guard.release();
12120    }
12121    Ok((outcomes, retried))
12122}
12123
12124/// Poll `pids` — commands [`run_commands`] reports as still running when its
12125/// own timeout elapsed — until every one is confirmed gone, or
12126/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
12127///
12128/// Real confirmation where confirmation is possible, not a substitute for
12129/// full process-tree observation: a grandchild the timed-out process spawned
12130/// and that survives independently of it is invisible to a pid check the
12131/// same way it always was, and continuing to observe and collect *that*
12132/// stays a different piece of work with its own owner. This only narrows a
12133/// fixed blind wait into an actual check of the pids this process does know
12134/// about.
12135async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
12136    wait_for_pids_with(
12137        pids,
12138        crate::proc::pid_alive,
12139        LEASE_RELEASE_POLL,
12140        LEASE_RELEASE_MAX_WAIT,
12141    )
12142    .await;
12143}
12144
12145/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
12146/// interval and ceiling supplied by the caller, so the polling *logic* -
12147/// returns as soon as every pid reports dead, gives up at the ceiling
12148/// otherwise - is testable on millisecond durations without asking the real
12149/// OS about a pid at all.
12150async fn wait_for_pids_with<F: Fn(u32) -> bool>(
12151    pids: &[u32],
12152    alive: F,
12153    poll: Duration,
12154    max_wait: Duration,
12155) {
12156    let deadline = Instant::now() + max_wait;
12157    loop {
12158        if pids.iter().all(|&pid| !alive(pid)) {
12159            return;
12160        }
12161        if Instant::now() >= deadline {
12162            return;
12163        }
12164        tokio::time::sleep(poll).await;
12165    }
12166}
12167
12168/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
12169/// admission that it could not even get a verify command to run, as opposed
12170/// to evidence the command actually produced? A caller that would otherwise
12171/// read a resource-blocked outcome as a red command must check this first:
12172/// see [`Runner::gate`], which retries rather than records `Blocked` when
12173/// this is true.
12174fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
12175    outcomes.iter().any(|o| o.resource_blocked)
12176}
12177
12178/// What [`Runner::gate_fix_round`] decided.
12179pub(crate) enum GateFix {
12180    /// The tree changed and `verify.e2e` is still green: run the gate again.
12181    Retry,
12182    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
12183    /// last failure stands and the run ends blocked.
12184    Stop,
12185    /// `verify.e2e` could not run after the fix (magi's own contention):
12186    /// decide nothing now, a later reentry retries.
12187    Defer,
12188}
12189
12190/// Is every red command in `outcomes` an ordinary failure the code could
12191/// explain: it ran, exited non-zero, and said something?
12192///
12193/// A timeout, a spawn failure and a killed process all leave `code` `None`;
12194/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
12195/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
12196/// what the command is stays the gate's business.
12197pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
12198    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12199    red.peek().is_some()
12200        && red.all(|o| {
12201            !o.resource_blocked
12202                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12203                && !o.output_tail.trim().is_empty()
12204        })
12205}
12206
12207/// Describe one verify command's outcome for the event log, distinguishing a
12208/// build/link failure — the toolchain never produced a binary to run — from
12209/// an actual test failure, since only the latter is a verdict on the patch.
12210fn e2e_outcome_label(o: &CommandOutcome) -> String {
12211    if o.ok() {
12212        return "pass".to_owned();
12213    }
12214    let reason = if o.build_failed() {
12215        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
12216    } else {
12217        format!("FAIL ({:?})", o.code)
12218    };
12219    format!(
12220        "{reason}\n{}",
12221        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12222    )
12223}
12224
12225/// Does `outcomes` hold a failure worth one more run on the same commit?
12226///
12227/// Only an ordinary test result qualifies: every red command ran, exited
12228/// non-zero and said which way (not a build/link failure, a timeout or kill,
12229/// a spawn failure, the shell's own 126 / 127, or a resource block). Those are
12230/// evidence about the machine or the toolchain, never about a flaky test.
12231fn rerunnable_failure(outcomes: &[CommandOutcome]) -> bool {
12232    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
12233    red.peek().is_some()
12234        && !verify_inconclusive(outcomes)
12235        && red.all(|o| {
12236            !o.build_failed() && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
12237        })
12238}
12239
12240/// Said at the top of the output a fixer gets when the same command failed on
12241/// both attempts, so it reads as a reproduced failure and not a one-off.
12242const FAILED_TWICE: &str = "[this command failed twice on the same commit; the output below is \
12243                            the second attempt's]";
12244
12245/// Run `verify.e2e`, retrying once if the first attempt could not build or
12246/// link — a build/link failure is frequently a race against a shared
12247/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
12248/// `verify` event per command, tagged with `context` (normally `"round N"`)
12249/// so the call sites that need this — the ordinary per-round leg in
12250/// `review_loop`, the deferred catch-up run `stop_reviewing` makes before it
12251/// will ever call a round green, `verify_current_head` and the gate-repair
12252/// check — read identically in the event log.
12253///
12254/// A failure that is a test result (see [`rerunnable_failure`]) is believed
12255/// only after the failed commands have run once more on the same commit, in
12256/// the same worktree and — because the caller holds it around this whole
12257/// function — under the same cache lease, with a full `timeout` of their own
12258/// (`[graph] rerun_failed_e2e`). A command that passes the second time is a
12259/// flake: the head is verified, the first failure is recorded and filed
12260/// (`crate::flake`, `RunState::verify_flakes`). A command that fails again
12261/// stands, carrying the second attempt's output. One verification is therefore
12262/// run at most three times: the build/link retry, then this one rerun.
12263/// `round` is the review round the verification belongs to, if any.
12264async fn run_e2e_with_retry(
12265    state: &mut RunState,
12266    shell: &[String],
12267    commands: &[String],
12268    worktree: &Path,
12269    timeout: Duration,
12270    context: &str,
12271    round: Option<usize>,
12272) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
12273    let (mut e2e, mut timed_out_pids) = run_commands(
12274        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
12275    )
12276    .await;
12277    for o in &e2e {
12278        state.event(
12279            "verify",
12280            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
12281        );
12282    }
12283    // A build/link failure is not a verdict on the patch — it is frequently a
12284    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
12285    // one retry before letting a red like that decide the round.
12286    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
12287    if verify_retried {
12288        state.event(
12289            "verify",
12290            format!(
12291                "{context}: verify could not build/link, not a test result — retrying once \
12292                 before concluding"
12293            ),
12294        );
12295        let retried = run_commands(
12296            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
12297        )
12298        .await;
12299        e2e = retried.0;
12300        // Both attempts' timeouts matter, not just the last one: the first
12301        // attempt's descendants may still be alive alongside the retry's.
12302        timed_out_pids.extend(retried.1);
12303        for o in &e2e {
12304            state.event(
12305                "verify",
12306                format!(
12307                    "{context}: retry `{}` -> {}",
12308                    o.command,
12309                    e2e_outcome_label(o)
12310                ),
12311            );
12312        }
12313    }
12314    if state.config.graph.rerun_failed_e2e && rerunnable_failure(&e2e) {
12315        let failed: Vec<usize> = (0..e2e.len()).filter(|&i| !e2e[i].ok()).collect();
12316        let rerun_commands: Vec<String> = failed.iter().map(|&i| e2e[i].command.clone()).collect();
12317        let head = git::rev_parse(worktree, "HEAD").await.unwrap_or_default();
12318        state.event(
12319            "verify",
12320            format!(
12321                "{context}: {} failed with a test result — running it once more on the same \
12322                 commit {} before deciding",
12323                rerun_commands
12324                    .iter()
12325                    .map(|c| format!("`{c}`"))
12326                    .collect::<Vec<_>>()
12327                    .join(", "),
12328                short(&head)
12329            ),
12330        );
12331        let (second, pids) = run_commands(
12332            state,
12333            "verify",
12334            "e2e",
12335            1 + usize::from(verify_retried),
12336            shell,
12337            &rerun_commands,
12338            worktree,
12339            timeout,
12340        )
12341        .await;
12342        timed_out_pids.extend(pids);
12343        for o in &second {
12344            state.event(
12345                "verify",
12346                format!(
12347                    "{context}: rerun `{}` -> {}",
12348                    o.command,
12349                    e2e_outcome_label(o)
12350                ),
12351            );
12352        }
12353        let mut reproduced = false;
12354        for (slot, &i) in failed.iter().enumerate() {
12355            let Some(again) = second.get(slot) else {
12356                break;
12357            };
12358            let first = std::mem::replace(&mut e2e[i], again.clone());
12359            if again.ok() {
12360                let signature = crate::flake::signature(&first.output_tail, first.code);
12361                let flake = crate::run::VerifyFlake {
12362                    round,
12363                    context: context.to_owned(),
12364                    head: head.clone(),
12365                    command: first.command.clone(),
12366                    first_failure: summarize_verify(&first.output_tail, EVENT_OUTPUT_TAIL),
12367                    fingerprint: crate::flake::fingerprint(&state.repo, &signature),
12368                    at: Timestamp::now(),
12369                    task: None,
12370                    filed: false,
12371                    filing_error: None,
12372                };
12373                crate::flake::record(state, flake, &signature.label, &crate::queue::Queue::open());
12374            } else if !again.resource_blocked {
12375                reproduced = true;
12376                e2e[i].output_tail = format!("{FAILED_TWICE}\n{}", again.output_tail);
12377            }
12378        }
12379        if reproduced {
12380            state.event(
12381                "verify",
12382                format!("{context}: the failure reproduced on the second run — it stands"),
12383            );
12384        }
12385    }
12386    (e2e, verify_retried, timed_out_pids)
12387}
12388
12389/// Run configured shell commands in `cwd`, in order. The second element is
12390/// the pid of every command that hit `timeout` and was still running when
12391/// this stopped waiting on it (best-effort: `None` when the platform did not
12392/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12393/// that releases a shared resource afterward needs to know.
12394///
12395/// Records `task` into [`RunState::active`] at every command boundary
12396/// (`RunState::task_command`) and clears it once the whole list has run
12397/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12398/// for minutes with no seat and no output of its own to show for it (see
12399/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12400/// apart from "not yet run" without this), and this is the only place that
12401/// knows which command is running right now and how many are left. Three
12402/// saves per command — start, not per second — matching the same "only at a
12403/// boundary" rule [`wave`] already follows for seats.
12404#[allow(clippy::too_many_arguments)]
12405async fn run_commands(
12406    state: &mut RunState,
12407    node: &str,
12408    task: &str,
12409    attempt: usize,
12410    shell: &[String],
12411    commands: &[String],
12412    cwd: &Path,
12413    timeout: Duration,
12414) -> (Vec<CommandOutcome>, Vec<u32>) {
12415    let isolated = crate::build_target::isolated(&state.config);
12416    if isolated && let Some(block) = &state.resource_block {
12417        return (
12418            vec![CommandOutcome {
12419                command: "(isolated build admission)".into(),
12420                code: None,
12421                output_tail: block.description(),
12422                duration_ms: 0,
12423                resource_blocked: true,
12424            }],
12425            Vec::new(),
12426        );
12427    }
12428    if commands.is_empty() {
12429        // Nothing to mark as running and nothing to clear — an empty list
12430        // means "not configured", and touching `active` (or the disk) over
12431        // that would be a write for every round of a repo with no
12432        // `verify.e2e` / `verify.gate` commands at all.
12433        return (Vec::new(), Vec::new());
12434    }
12435    let mut out = Vec::new();
12436    let mut timed_out_pids = Vec::new();
12437    let total = commands.len();
12438    for (idx, command) in commands.iter().enumerate() {
12439        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12440        if let Err(e) = state.save() {
12441            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12442        }
12443        let started = Instant::now();
12444        let mut cmd = tokio::process::Command::new(&shell[0]);
12445        cmd.quiet();
12446        // Verify the checkout as a standalone caller. Worker ownership and
12447        // pinned executable selection belong to this driver, not to tests or
12448        // tools it runs. Preserve home/seat context and Running's containment.
12449        for key in crate::worker::INVOCATION_ENV {
12450            cmd.env_remove(key);
12451        }
12452        cmd.args(&shell[1..])
12453            .current_dir(cwd)
12454            .stdin(std::process::Stdio::null())
12455            .stdout(std::process::Stdio::piped())
12456            .stderr(std::process::Stdio::piped())
12457            .kill_on_drop(true);
12458        let mut outputs = Vec::new();
12459        let mut cache = None;
12460        let home = crate::run::home();
12461        let head = if isolated {
12462            Some(git::rev_parse(cwd, "HEAD").await)
12463        } else {
12464            None
12465        };
12466        let protected = (|| -> Result<()> {
12467            crate::incremental::configure(&mut cmd, cwd)?;
12468            cache = if isolated {
12469                let target = VERIFY_TARGET
12470                    .try_with(Clone::clone)
12471                    .context("isolated verification has no target lease")?;
12472                cmd.arg(crate::build_target::command::normalize(command)?);
12473                crate::build_target::command::configure(&mut cmd, &target)?;
12474                Some(target)
12475            } else {
12476                crate::build_target::command::clear(&mut cmd)?;
12477                cmd.arg(command);
12478                state.config.cache_dir()
12479            };
12480            let mut paths = vec![cwd.to_owned()];
12481            if let Some(cache) = &cache {
12482                paths.push(cache.clone());
12483            }
12484            let admit = if isolated {
12485                crate::retention::OutputLease::admit_isolated
12486            } else {
12487                crate::retention::OutputLease::admit
12488            };
12489            outputs.extend(admit(&home, &paths, state.config.disk.min_free_bytes)?);
12490            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12491                outputs.push(guard);
12492            }
12493            if let Some(cache) = &cache {
12494                outputs.push(crate::retention::OutputLease::cache(
12495                    &crate::run::home(),
12496                    cache,
12497                    false,
12498                )?);
12499                if let Some(head) = head {
12500                    crate::build_target::ensure_fresh(&home, cache, cwd, &head?, &state.config)?;
12501                    if task == "pre_gate" {
12502                        crate::cache::invalidate_identity(&home, cache);
12503                    }
12504                }
12505            }
12506            Ok(())
12507        })();
12508        if let Err(e) = protected {
12509            let detail = format!("execution admission unavailable: {e:#}");
12510            state.resource_block = Some(crate::run::ResourceBlock {
12511                cause: crate::run::ResourceBlockCause::Other,
12512                node: node.to_owned(),
12513                kind: if detail.contains("disk admission") {
12514                    "disk admission"
12515                } else {
12516                    "output ownership"
12517                }
12518                .to_owned(),
12519                detail: detail.clone(),
12520                at: Timestamp::now(),
12521            });
12522            out.push(CommandOutcome {
12523                command: command.clone(),
12524                code: None,
12525                output_tail: detail,
12526                duration_ms: 0,
12527                resource_blocked: true,
12528            });
12529            break;
12530        }
12531        let spawned = if isolated {
12532            crate::seat_process::Running::spawn_build(cmd, None, outputs)
12533        } else {
12534            crate::seat_process::Running::spawn(cmd, None, outputs)
12535        };
12536        let (code, body) = match spawned {
12537            Ok(mut running) => {
12538                let pid = running.child().id();
12539                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12540                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12541                let mut paths = vec![cwd.to_owned()];
12542                if let Some(cache) = cache {
12543                    paths.push(cache);
12544                }
12545                let mut pressure = None;
12546                let status = tokio::select! {
12547                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12548                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes, isolated) => {
12549                        pressure = Some(reason); None
12550                    }
12551                };
12552                let (code, failure) = match status {
12553                    Some(Ok(Ok(status))) => (
12554                        status.code(),
12555                        status
12556                            .code()
12557                            .is_none()
12558                            .then(|| format!("command terminated: {status}")),
12559                    ),
12560                    Some(Ok(Err(error))) => {
12561                        (None, Some(format!("failed waiting for command: {error}")))
12562                    }
12563                    Some(Err(_)) => {
12564                        // Leased cleanup confirms process-tree death itself.
12565                        // Unleased commands still need the caller's PID check
12566                        // after kill-on-drop, before releasing cache ownership.
12567                        if !running.coordinated()
12568                            && let Some(pid) = pid
12569                        {
12570                            timed_out_pids.push(pid);
12571                        }
12572                        (
12573                            None,
12574                            Some(format!("timed out after {}s", timeout.as_secs())),
12575                        )
12576                    }
12577                    None => (None, None),
12578                };
12579                running.finish().await;
12580                if let Some(reason) = pressure {
12581                    state.resource_block = Some(crate::run::ResourceBlock {
12582                        cause: crate::run::ResourceBlockCause::Other,
12583                        node: node.to_owned(),
12584                        kind: "disk pressure".to_owned(),
12585                        detail: reason.clone(),
12586                        at: Timestamp::now(),
12587                    });
12588                    out.push(CommandOutcome {
12589                        command: command.clone(),
12590                        code: None,
12591                        output_tail: reason,
12592                        duration_ms: started.elapsed().as_millis() as u64,
12593                        resource_blocked: true,
12594                    });
12595                    break;
12596                }
12597                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12598                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12599                if let Some(failure) = failure {
12600                    body.push('\n');
12601                    body.push_str(&failure);
12602                }
12603                (code, body)
12604            }
12605            Err(e) if isolated => {
12606                let e = anyhow::Error::new(e)
12607                    .context(format!("could not start managed build `{}`", shell[0]));
12608                record_cache_error(state, node, &e);
12609                out.push(CommandOutcome {
12610                    command: command.clone(),
12611                    code: None,
12612                    output_tail: format!("{e:#}"),
12613                    duration_ms: 0,
12614                    resource_blocked: true,
12615                });
12616                break;
12617            }
12618            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12619        };
12620        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12621        if !body.is_empty() {
12622            // The full text is kept whatever the summary dropped. A fresh
12623            // name per command so a resume or retry never overwrites an
12624            // earlier run's output. A failed write changes no verdict.
12625            match crate::run::write_verification_artifact(state, task, &body) {
12626                Ok(name) if summary != body => {
12627                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12628                }
12629                Ok(_) => {}
12630                Err(e) => {
12631                    let warning = format!("could not save the full {task} output: {e:#}");
12632                    tracing::warn!("{warning}");
12633                    state.event(node, warning);
12634                    if summary != body {
12635                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12636                    }
12637                }
12638            }
12639        }
12640        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12641        out.push(CommandOutcome {
12642            command: command.clone(),
12643            code,
12644            output_tail: summary,
12645            duration_ms: started.elapsed().as_millis() as u64,
12646            resource_blocked,
12647        });
12648        if resource_blocked {
12649            state.resource_block = Some(crate::run::ResourceBlock {
12650                cause: crate::run::ResourceBlockCause::Other,
12651                node: node.to_owned(),
12652                kind: "incremental cache admission".into(),
12653                detail: body,
12654                at: Timestamp::now(),
12655            });
12656            break;
12657        }
12658    }
12659    state.task_finished(task);
12660    if let Err(e) = state.save() {
12661        tracing::warn!("could not persist the end of {task}: {e:#}");
12662    }
12663    (out, timed_out_pids)
12664}
12665
12666/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12667/// section (`report::run`) and in the `merge` event this node records — for
12668/// the operator to run by hand.
12669///
12670/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12671/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12672/// merge commits", or "require linear history") rejects the push a `--no-ff`
12673/// merge would produce, which is exactly the guidance this function replaces.
12674/// `message`'s first line becomes the squash commit's subject, matching the
12675/// note `report::run` prints alongside this command — see that function for
12676/// why an explicit subject is not optional there.
12677fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12678    let repo = repo.display();
12679    match style {
12680        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12681        MergeStyle::Squash => {
12682            // The subject sits inside double quotes, and a title an agent
12683            // wrote may carry the characters that break out of them.
12684            let subject = message
12685                .lines()
12686                .next()
12687                .unwrap_or(branch)
12688                .replace(['\\', '"', '$', '`'], "");
12689            format!(
12690                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12691            )
12692        }
12693        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12694    }
12695}
12696
12697/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12698/// under the hood, rejects a `title` over 256 characters and the whole
12699/// command fails — no PR at all, for a run whose body was otherwise fine
12700/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12701/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12702/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12703/// this bound. It is a margin, not a guarantee — a title packed
12704/// with multi-unit characters could still in principle land close to the
12705/// edge, but a real task title's occasional emoji or accented letter fits
12706/// comfortably inside it.
12707const PR_TITLE_MAX: usize = 240;
12708
12709/// A pull request title from the opening line of `text`, or `None` when that
12710/// line is not English (GitHub text is) or has no letters.
12711///
12712/// A line within `max` is kept as is. A longer one is cut at the end of its
12713/// first sentence when that falls inside `max`, else at a word boundary with a
12714/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12715/// make every merely-truncated title look non-English). The language check
12716/// runs on the kept text before any mark is added, so only what GitHub will
12717/// show is judged: an English opening followed by non-ASCII far past the cut
12718/// still passes.
12719fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12720    let line = queue::first_line(text)?;
12721    let chars: Vec<char> = line.chars().collect();
12722    let (kept, mark) = if chars.len() <= max {
12723        (line.to_owned(), "")
12724    } else if let Some(end) = sentence_end(&chars, max) {
12725        (chars[..end].iter().collect::<String>(), "")
12726    } else {
12727        let room = max.saturating_sub(3);
12728        // Cut at the last space inside the room; when the char just past the
12729        // room is a space the room already ends on a word.
12730        let cut = if chars[room].is_whitespace() {
12731            room
12732        } else {
12733            chars[..room]
12734                .iter()
12735                .rposition(|c| c.is_whitespace())
12736                .unwrap_or(room)
12737        };
12738        let head: String = chars[..cut].iter().collect();
12739        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12740        (head.to_owned(), "...")
12741    };
12742    if kept.is_empty() {
12743        return None;
12744    }
12745    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12746        return None;
12747    }
12748    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12749        return None;
12750    }
12751    Some(format!("{kept}{mark}"))
12752}
12753
12754/// The char length of the first sentence of `chars` when it ends within `max`
12755/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12756/// abbreviations so `e.g. foo` does not end a title early.
12757fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12758    const MIN: usize = 20;
12759    for i in MIN..max.min(chars.len()) {
12760        if !matches!(chars[i], '.' | '!' | '?') {
12761            continue;
12762        }
12763        let Some(&next) = chars.get(i + 1) else {
12764            continue;
12765        };
12766        if !next.is_whitespace() {
12767            continue;
12768        }
12769        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12770        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12771            continue;
12772        }
12773        let word: String = chars[..i]
12774            .iter()
12775            .rev()
12776            .take_while(|c| !c.is_whitespace())
12777            .collect::<Vec<_>>()
12778            .into_iter()
12779            .rev()
12780            .collect();
12781        let word = word.to_ascii_lowercase();
12782        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12783            continue;
12784        }
12785        let end = chars[..i]
12786            .iter()
12787            .rposition(|c| !c.is_whitespace())
12788            .map_or(i, |p| p + 1);
12789        return Some(end);
12790    }
12791    None
12792}
12793
12794/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12795/// change: a title and a body describing what was *implemented*, not the task
12796/// that asked for it. A task reads as a request; a reader of the merged
12797/// history wants the change.
12798struct PrMessage {
12799    title: String,
12800    body: String,
12801}
12802
12803impl PrMessage {
12804    /// Title, blank line, body. The first line is the squash/merge commit
12805    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12806    /// has to stay one sensible line.
12807    fn commit_message(&self) -> String {
12808        format!("{}\n\n{}", self.title, self.body)
12809    }
12810}
12811
12812/// The text after a leading `TITLE:` (any case) on `line`.
12813fn title_marker(line: &str) -> Option<&str> {
12814    let line = line.trim();
12815    let head = line.get(..6)?;
12816    head.eq_ignore_ascii_case("title:")
12817        .then(|| line[6..].trim())
12818}
12819
12820/// The implementer's own one-line title: the `TITLE:` line the implement
12821/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12822/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12823/// source, and a title that says as much is refused here too.
12824fn summary_title(summary: &str) -> Option<String> {
12825    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12826    let raw = title_marker(first)?;
12827    if raw.is_empty() {
12828        return None;
12829    }
12830    let title = queue::title_from(raw, PR_TITLE_MAX);
12831    let lower = title.to_ascii_lowercase();
12832    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12833        return None;
12834    }
12835    Some(title)
12836}
12837
12838/// How `open_review`'s instruction begins; see [`landing_title`].
12839const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12840
12841/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12842pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12843
12844/// Does this run review work that already existed, rather than implement a
12845/// task? Runs recorded before `reviewed_commits` existed carry only the
12846/// candidate marker.
12847pub(crate) fn is_review_run(state: &RunState) -> bool {
12848    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12849}
12850
12851/// The title of a review-only run: the subject of the oldest commit under
12852/// review. Later commits are usually fixups, and `instruction` is the review
12853/// prompt, which says nothing about the change. GitHub text is English, so a
12854/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12855fn review_title(state: &RunState) -> Option<String> {
12856    english_subject(
12857        state.reviewed_commits.as_ref()?.first()?,
12858        state.config.graph.github_english(),
12859    )
12860}
12861
12862/// `raw` as a pull request title, or `None` when it is blank, not English
12863/// (GitHub text is), or one of magi's own candidate commit subjects.
12864fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12865    let raw = raw.trim();
12866    if raw.is_empty() {
12867        return None;
12868    }
12869    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12870        return None;
12871    }
12872    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12873        return None;
12874    }
12875    let title = queue::title_from(raw, PR_TITLE_MAX);
12876    let lower = title.to_ascii_lowercase();
12877    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12878        return None;
12879    }
12880    Some(title)
12881}
12882
12883/// What a review-only run's branch says about itself, read at the moment the
12884/// pull request is opened.
12885#[derive(Debug, Clone, PartialEq, Eq)]
12886struct BranchFacts {
12887    /// `(subject, body)` of each commit, oldest first.
12888    commits: Vec<(String, String)>,
12889    /// Trimmed `git diff --stat`.
12890    stat: String,
12891}
12892
12893/// Longest diff stat shown: this many file lines plus the summary line.
12894const STAT_FILE_LINES: usize = 25;
12895/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12896const COMMITS_MAX_CHARS: usize = 20_000;
12897
12898/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12899/// say or finds nothing, so the caller falls back to what the run recorded.
12900async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12901    let commits = git::commit_log(repo, base, branch).await.ok()?;
12902    if commits.is_empty() {
12903        return None;
12904    }
12905    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12906    let lines: Vec<&str> = stat.lines().collect();
12907    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12908        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12909        let more = format!(" ... {omitted} more file(s)");
12910        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12911        kept.push(&more);
12912        kept.push(lines[lines.len() - 1]);
12913        kept.join("\n")
12914    } else {
12915        lines.join("\n")
12916    };
12917    Some(BranchFacts { commits, stat })
12918}
12919
12920/// Defang what would break the surrounding markdown: a closing `</details>`
12921/// and a code fence.
12922fn markdown_safe(text: &str) -> String {
12923    text.replace("</details>", "&lt;/details&gt;")
12924        .replace("\x60\x60\x60", "~~~")
12925}
12926
12927fn neutral_title(state: &RunState, winner: char) -> String {
12928    format!(
12929        "chore: land candidate {} of run {}",
12930        winner.to_ascii_uppercase(),
12931        state.id
12932    )
12933}
12934
12935/// Reuse the graph's bounded fixer without starting another graph or driver.
12936/// Copy the working state back even on errors so spent rounds remain visible.
12937pub(crate) async fn repair_landing_gate(
12938    state: &mut RunState,
12939    winner: &Candidate,
12940    outcomes: &[CommandOutcome],
12941) -> Result<GateFix> {
12942    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12943        return Ok(GateFix::Stop);
12944    }
12945    let mut runner = Runner {
12946        roles: state.config.resolve_roles()?,
12947        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12948        state: state.clone(),
12949        pause: Pause::new(),
12950        interrupt: Pause::new(),
12951    };
12952    let result = runner.gate_fix_round(winner, outcomes).await;
12953    *state = runner.state;
12954    result
12955}
12956
12957/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12958/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12959pub(crate) async fn verify_current_head(
12960    state: &mut RunState,
12961    winner: &Candidate,
12962    wait: &crate::repo_lock::WaitPolicy,
12963) -> Result<bool> {
12964    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12965    // Preserve the previous record before replacing it, including on crash recovery.
12966    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12967        state
12968            .verification_history
12969            .push(crate::run::VerificationEvidence {
12970                superseded_by: Some(head.clone()),
12971                archived_at: Some(Timestamp::now()),
12972                fixes_spent: Some(state.gate_fixes.len()),
12973                gate_head: None,
12974                gate_ran: false,
12975                gate: Vec::new(),
12976                e2e: state.head_e2e.take(),
12977            });
12978    }
12979    if let Some(e) = &state.head_e2e
12980        && e.head == head
12981    {
12982        if e.passed {
12983            return Ok(true);
12984        }
12985        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12986            state.status = RunStatus::Blocked;
12987            state.save()?;
12988            return Ok(false);
12989        }
12990    } else if state.reviews.last().is_some_and(|r| {
12991        r.verified_head.as_deref() == Some(head.as_str())
12992            && (r.e2e_status() == E2eStatus::Passed
12993                || (r.e2e_status() == E2eStatus::NotConfigured
12994                    && state.config.verify.e2e.is_empty()))
12995    }) || state
12996        .gate_repair_e2e
12997        .as_ref()
12998        .is_some_and(|e| e.head == head && e.passed)
12999    {
13000        return Ok(true);
13001    }
13002    if let Some(e) = &state.gate_repair_e2e
13003        && e.head == head
13004        && !e.passed
13005        && !e.outcomes.is_empty()
13006        && !verify_inconclusive(&e.outcomes)
13007    {
13008        state.status = RunStatus::Blocked;
13009        state.save()?;
13010        return Ok(false);
13011    }
13012    state.head_e2e = Some(crate::run::GateRepairE2e {
13013        head: head.clone(),
13014        passed: false,
13015        outcomes: Vec::new(),
13016    });
13017    state.save()?;
13018    let commands = state.config.verify.e2e.clone();
13019    let shell = state.config.shell();
13020    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
13021    let cache_dir = state.config.cache_dir();
13022    let outcomes = if commands.is_empty() {
13023        Vec::new()
13024    } else {
13025        let (outcomes, _) = with_cache_lease(
13026            state,
13027            cache_dir.as_deref(),
13028            "e2e",
13029            "e2e",
13030            &winner.worktree,
13031            &head,
13032            timeout,
13033            "current head e2e",
13034            wait,
13035            |state, budget| async move {
13036                run_e2e_with_retry(
13037                    state,
13038                    &shell,
13039                    &commands,
13040                    &winner.worktree,
13041                    budget,
13042                    "current head e2e",
13043                    None,
13044                )
13045                .await
13046            },
13047        )
13048        .await?;
13049        outcomes
13050    };
13051    let unresolved = verify_inconclusive(&outcomes);
13052    let passed = !unresolved
13053        && outcomes.iter().all(CommandOutcome::ok)
13054        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
13055    if !outcomes.is_empty() && !unresolved {
13056        state.record_base_progress(&head);
13057    }
13058    state.head_e2e = Some(crate::run::GateRepairE2e {
13059        head: head.clone(),
13060        passed,
13061        outcomes,
13062    });
13063    if !passed && !unresolved {
13064        state.status = RunStatus::Blocked;
13065        state.event(
13066            "e2e",
13067            format!("verify.e2e failed on head {head}; nothing published"),
13068        );
13069    }
13070    state.save()?;
13071    Ok(passed)
13072}
13073
13074/// Recheck a repair's e2e requirement across failures, resource deferrals and
13075/// process interruptions. A successful result only applies to its exact SHA.
13076pub(crate) async fn check_gate_repair_e2e(
13077    state: &mut RunState,
13078    winner: &Candidate,
13079    wait: &crate::repo_lock::WaitPolicy,
13080) -> Result<GateFix> {
13081    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
13082        return Ok(GateFix::Retry);
13083    }
13084    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
13085    if state
13086        .gate_repair_e2e
13087        .as_ref()
13088        .is_some_and(|r| r.head == head && r.passed)
13089    {
13090        return Ok(GateFix::Retry);
13091    }
13092    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
13093        head: head.clone(),
13094        passed: false,
13095        outcomes: Vec::new(),
13096    });
13097    state.save()?;
13098    let commands = state.config.verify.e2e.clone();
13099    let outcomes = if commands.is_empty() {
13100        Vec::new()
13101    } else {
13102        let shell = state.config.shell();
13103        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
13104        let cache_dir = state.config.cache_dir();
13105        let (outcomes, _) = with_cache_lease(
13106            state,
13107            cache_dir.as_deref(),
13108            "e2e",
13109            "e2e",
13110            &winner.worktree,
13111            &head,
13112            timeout,
13113            "gate repair e2e",
13114            wait,
13115            |state, budget| async move {
13116                run_e2e_with_retry(
13117                    state,
13118                    &shell,
13119                    &commands,
13120                    &winner.worktree,
13121                    budget,
13122                    "gate repair e2e",
13123                    None,
13124                )
13125                .await
13126            },
13127        )
13128        .await?;
13129        outcomes
13130    };
13131    let deferred = verify_inconclusive(&outcomes);
13132    let passed = outcomes.iter().all(CommandOutcome::ok)
13133        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
13134    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
13135        head,
13136        passed,
13137        outcomes,
13138    });
13139    state.save()?;
13140    if deferred {
13141        Ok(GateFix::Defer)
13142    } else if !passed {
13143        state.event(
13144            "gate",
13145            "verify.e2e failed after gate repair; nothing published",
13146        );
13147        Ok(GateFix::Stop)
13148    } else {
13149        Ok(GateFix::Retry)
13150    }
13151}
13152
13153/// Run the same configured gate for a landing head and a fresh graph.
13154pub(crate) async fn run_landing_gate(
13155    state: &mut RunState,
13156    winner: &Candidate,
13157    wait: &crate::repo_lock::WaitPolicy,
13158) -> Result<Vec<CommandOutcome>> {
13159    let shell = state.config.shell();
13160    let gate_commands = state.config.verify.gate.clone();
13161    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
13162    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
13163        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
13164        Vec::new()
13165    } else {
13166        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
13167        let cache_dir = state.config.cache_dir();
13168        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
13169        let (outcomes, _) = with_cache_lease(
13170            state,
13171            cache_dir.as_deref(),
13172            "gate",
13173            "gate",
13174            &winner.worktree,
13175            &head,
13176            timeout,
13177            "final gate",
13178            wait,
13179            |state, budget| {
13180                let shell = shell.clone();
13181                let gate_commands = gate_commands.clone();
13182                let winner = winner.clone();
13183                async move {
13184                    Runner::run_pre_gate(state, &winner).await;
13185                    // Bind command evidence after mechanical fixes, before commands start.
13186                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
13187                    let (outcomes, timed_out_pids) = run_commands(
13188                        state,
13189                        "gate",
13190                        "gate",
13191                        0,
13192                        &shell,
13193                        &gate_commands,
13194                        &winner.worktree,
13195                        budget,
13196                    )
13197                    .await;
13198                    (outcomes, false, timed_out_pids)
13199                }
13200            },
13201        )
13202        .await?;
13203        outcomes
13204    };
13205    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
13206    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
13207        let mut outcomes = outcomes;
13208        outcomes.push(CommandOutcome {
13209            command: "gate head stability".into(), code: None,
13210            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
13211            duration_ms: 0, resource_blocked: false,
13212        });
13213        return Ok(outcomes);
13214    }
13215    if outcomes.is_empty() {
13216        // Nothing configured to check — distinct from every other
13217        // silence in this run's event log, since an empty `gate` alone
13218        // no longer says whether the gate ran at all (see
13219        // `RunState::gate_ran`'s own doc).
13220        state.event("gate", "no gate commands configured; not tested");
13221    }
13222    for o in &outcomes {
13223        state.event(
13224            "gate",
13225            format!(
13226                "`{}` -> {}",
13227                o.command,
13228                if o.ok() {
13229                    "pass".to_owned()
13230                } else {
13231                    format!(
13232                        "FAIL ({:?})\n{}",
13233                        o.code,
13234                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
13235                    )
13236                }
13237            ),
13238        );
13239    }
13240    Ok(outcomes)
13241}
13242
13243/// The pull request title to hand to `land::merge_subject`. A review-only run
13244/// opened by an earlier build titled its pull request with the review prompt;
13245/// that title is dropped (empty, so the fallback applies) rather than landed.
13246/// Any other title, including an operator's rename, passes through untouched,
13247/// and so does every title of a run that implements a task.
13248pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
13249    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
13250        ""
13251    } else {
13252        pr_title
13253    }
13254}
13255
13256/// What the squash subject falls back to when the pull request title is empty
13257/// or candidate-shaped: for a review-only run the derived title, never the
13258/// review prompt held in `instruction`.
13259pub fn landing_subject_source(state: &RunState) -> String {
13260    if is_review_run(state) {
13261        let winner = state.candidates.first().map_or('A', |c| c.label);
13262        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
13263    }
13264    state.instruction.clone()
13265}
13266
13267/// `summary` without its `TITLE:` line, which the pull request title already
13268/// carries.
13269fn summary_without_title(summary: &str) -> String {
13270    let mut lines = summary.trim().lines().peekable();
13271    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
13272        lines.next();
13273    }
13274    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
13275}
13276
13277/// The pull request title and body for the winning candidate.
13278///
13279/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
13280/// the task's own opening line via [`queue::title_from`] when there is none.
13281/// `state.instruction` can open with blank lines (`task_text` only rejects a
13282/// body that is blank *entirely*), which `title_from` skips.
13283///
13284/// Publishing uses an independent description generated from the final diff
13285/// and structured evidence. Internal summaries and reports remain local.
13286/// Non-publishing previews may display the original task and candidate summary.
13287/// The footer retains the run and candidate provenance.
13288#[cfg(test)]
13289fn pr_message(state: &RunState, winner: char) -> PrMessage {
13290    pr_message_with(state, winner, None)
13291}
13292
13293/// [`pr_message`] with what the branch of a review-only run says about itself.
13294/// `facts` is ignored for a run that implements a task.
13295#[cfg(test)]
13296fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
13297    let raw = pr_message_raw(
13298        state,
13299        winner,
13300        facts,
13301        crate::pr_outcome::FinalHead::Unreported,
13302        ChangeNote::NotAsked,
13303    );
13304    let id = crate::scrub::Identity::current();
13305    PrMessage {
13306        title: crate::scrub::scrub(&raw.title, &id),
13307        body: crate::scrub::scrub(&raw.body, &id),
13308    }
13309}
13310
13311/// What the public description may say about the change itself.
13312#[derive(Debug, Clone, Copy)]
13313enum ChangeNote<'a> {
13314    /// A local preview or non-publishing merge mode: no public generation needed.
13315    NotAsked,
13316    /// A summary written from the final diff.
13317    Summary(&'a crate::run::ChangeSummary),
13318    /// A summary was asked for and could not be produced; the text says why.
13319    Unavailable(&'a str),
13320}
13321
13322fn pr_message_raw(
13323    state: &RunState,
13324    winner: char,
13325    facts: Option<&BranchFacts>,
13326    head: crate::pr_outcome::FinalHead<'_>,
13327    change: ChangeNote<'_>,
13328) -> PrMessage {
13329    if !matches!(change, ChangeNote::NotAsked) {
13330        let (title, mut body) = match change {
13331            ChangeNote::Summary(c) => (
13332                c.title.clone(),
13333                format!("## Summary\n\n{}\n\n", c.body.trim()),
13334            ),
13335            ChangeNote::Unavailable(why) => (
13336                neutral_title(state, winner),
13337                format!(
13338                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
13339                ),
13340            ),
13341            ChangeNote::NotAsked => unreachable!(),
13342        };
13343        body.push_str(&public_pr_evidence(state, head));
13344        body.push_str(&format!(
13345            "\n---\nmagi:run/{} magi:candidate-{}\n",
13346            state.id,
13347            winner.to_ascii_lowercase()
13348        ));
13349        return PrMessage { title, body };
13350    }
13351    pr_preview(state, winner, facts, head)
13352}
13353
13354/// Append public facts from the run records after every prose generation,
13355/// including posting-gate rewrites. The agent cannot drop this provenance.
13356fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
13357    let mut body = crate::pr_outcome::review_rounds(state);
13358    body.push_str(&crate::pr_outcome::final_verification(state, head));
13359    let open = state.last_round_findings();
13360    if !open.is_empty() {
13361        body.push_str("## Open review findings\n\n");
13362        for f in open {
13363            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
13364        }
13365        body.push('\n');
13366    }
13367    if let Some(c) = state.change_summary.as_ref()
13368        && head == crate::pr_outcome::FinalHead::At(&c.head)
13369    {
13370        body.push_str(&format!(
13371            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
13372            c.base, c.head
13373        ));
13374    }
13375    body
13376}
13377
13378/// Local-only preview of the operator's task and candidate summary.
13379fn pr_preview(
13380    state: &RunState,
13381    winner: char,
13382    facts: Option<&BranchFacts>,
13383    head: crate::pr_outcome::FinalHead<'_>,
13384) -> PrMessage {
13385    let summary = state
13386        .candidates
13387        .iter()
13388        .find(|c| c.label == winner)
13389        .map(|c| c.summary.as_str())
13390        .unwrap_or_default();
13391    // The fallback is the operator's own words and may not be English; GitHub
13392    // text always is, so a non-English task gets a neutral title instead.
13393    let review = is_review_run(state);
13394    let title = if review {
13395        facts
13396            .and_then(|f| {
13397                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
13398            })
13399            .or_else(|| review_title(state))
13400            .or_else(|| {
13401                state
13402                    .candidates
13403                    .iter()
13404                    .find(|c| c.label == winner)
13405                    .filter(|c| !c.branch.starts_with("magi/"))
13406                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
13407            })
13408            .unwrap_or_else(|| neutral_title(state, winner))
13409    } else {
13410        summary_title(summary).unwrap_or_else(|| {
13411            english_title(
13412                &state.instruction,
13413                PR_TITLE_MAX,
13414                state.config.graph.github_english(),
13415            )
13416            .unwrap_or_else(|| neutral_title(state, winner))
13417        })
13418    };
13419
13420    let mut body = String::new();
13421    let what = summary_without_title(summary);
13422    if !what.is_empty() {
13423        body.push_str("## Summary\n\n");
13424        body.push_str(&what);
13425        body.push_str("\n\n");
13426    }
13427
13428    // Review history uses structured facts across all rounds, never reports.
13429    body.push_str(&crate::pr_outcome::review_rounds(state));
13430    body.push_str(&crate::pr_outcome::final_verification(state, head));
13431    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13432
13433    let open = state.open_findings();
13434    if !open.is_empty() {
13435        body.push_str("## Open review findings\n\n");
13436        for f in &open {
13437            body.push_str(&format!(
13438                "- `{}` [{:?}] {}\n",
13439                f.id, f.severity, "Context retained in local review records."
13440            ));
13441        }
13442        body.push('\n');
13443    }
13444
13445    if let Some(fix) = fix
13446        && !fix.rejected.is_empty()
13447    {
13448        body.push_str("## Declined by the fixer\n\n");
13449        for r in &fix.rejected {
13450            body.push_str(&format!(
13451                "- `{}`: {}\n",
13452                r.id, "Reason retained in local review records."
13453            ));
13454        }
13455        body.push('\n');
13456    }
13457
13458    if review {
13459        // The review prompt is not the task; list what the branch carries.
13460        body.push_str("## Commits under review\n\n");
13461        if let Some(facts) = facts {
13462            let mut left = COMMITS_MAX_CHARS;
13463            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13464                let mut entry = format!("- {}\n", markdown_safe(subject));
13465                for l in markdown_safe(text).lines() {
13466                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13467                }
13468                if left == 0 {
13469                    body.push_str(&format!(
13470                        "- ... {} more commit(s)\n",
13471                        facts.commits.len() - i
13472                    ));
13473                    break;
13474                }
13475                if entry.len() > left {
13476                    // Even the first commit is cut: one huge body must not
13477                    // push the whole description past GitHub's limit.
13478                    let mut end = left;
13479                    while !entry.is_char_boundary(end) {
13480                        end -= 1;
13481                    }
13482                    entry.truncate(end);
13483                    entry.push_str("\n  ... (truncated)\n");
13484                    left = 0;
13485                } else {
13486                    left -= entry.len();
13487                }
13488                body.push_str(&entry);
13489            }
13490            if !facts.stat.trim().is_empty() {
13491                body.push_str(&format!(
13492                    "\n## Diff stat\n\n```\n{}\n```\n",
13493                    markdown_safe(facts.stat.trim())
13494                ));
13495            }
13496        } else {
13497            match &state.reviewed_commits {
13498                Some(subjects) => {
13499                    for s in subjects {
13500                        body.push_str(&format!("- {}\n", s.trim()));
13501                    }
13502                }
13503                None => {
13504                    // An older run kept only the prompt, with the commit list
13505                    // after its first paragraph.
13506                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13507                    body.push_str(rest.trim());
13508                    body.push('\n');
13509                }
13510            }
13511        }
13512    } else {
13513        let task = state.instruction.trim();
13514        let task = if task.is_empty() {
13515            "(empty task)"
13516        } else {
13517            task
13518        };
13519        body.push_str(&format!(
13520            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13521            task.replace("</details>", "&lt;/details&gt;")
13522        ));
13523    }
13524
13525    body.push_str(&format!(
13526        "\n---\nmagi:run/{} magi:candidate-{}\n",
13527        state.id,
13528        winner.to_ascii_lowercase()
13529    ));
13530
13531    PrMessage { title, body }
13532}
13533
13534/// Source imports have already been prepared from structured inputs. Legacy
13535/// inferred references must never instruct a seat to reconstruct those imports.
13536fn seeded_instruction(state: &RunState) -> String {
13537    let mut inputs = Vec::new();
13538    if let Some(start) = &state.sources.start_from {
13539        inputs.push(format!(
13540            "Start from `{}` at `{}` (including ancestry).",
13541            start.token, start.sha
13542        ));
13543    }
13544    for pick in &state.sources.pick_commits {
13545        inputs.push(format!(
13546            "Pick `{}` at `{}` (single commit).",
13547            pick.token, pick.sha
13548        ));
13549    }
13550    if inputs.is_empty() {
13551        state.instruction.clone()
13552    } else {
13553        format!(
13554            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13555            state.instruction,
13556            inputs.join("\n")
13557        )
13558    }
13559}
13560
13561/// Does the winner have no commits ahead of the base it would land on?
13562/// Any failure to find out reads as "not empty": the merge then behaves as it
13563/// always did rather than refusing on a guess.
13564async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13565    let base = &state.base_branch;
13566    let mut against = base.clone();
13567    if mode == MergeMode::Pr {
13568        // A pull request lands on the remote's base, never the local branch
13569        // of the same name: if that cannot be read, "not empty" is the safe
13570        // answer.
13571        let remote = &state.config.merge.remote;
13572        let tracking = format!("{remote}/{base}");
13573        let fetched = git::fetch(repo, remote, base).await;
13574        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13575            against = tracking;
13576        } else {
13577            return false;
13578        }
13579    }
13580    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13581}
13582
13583/// Why nothing was opened for an empty winner, with what the task's own
13584/// references resolved to.
13585fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13586    let mut detail = format!(
13587        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13588         nothing to open a pull request for"
13589    );
13590    detail.push_str(
13591        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13592    );
13593    detail
13594}
13595
13596/// What the `Pr` merge does once it knows whether the branch already has an
13597/// open pull request.
13598#[derive(Debug, PartialEq, Eq)]
13599enum PrPlan {
13600    Create,
13601    Adopt { url: String, title: String },
13602    Stop(String),
13603}
13604
13605/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13606/// a failed lookup -> stop with the real reason. Never guesses.
13607fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13608    match found {
13609        Ok(land::OpenPr::None) => PrPlan::Create,
13610        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13611        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13612            "several open pull requests exist for this branch, not picking one: {}",
13613            urls.join(" ")
13614        )),
13615        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13616    }
13617}
13618
13619/// `gh pr create`, returning the PR url.
13620pub(crate) async fn gh_pr_create(
13621    cwd: &Path,
13622    base: &str,
13623    head: &str,
13624    title: &str,
13625    body: &str,
13626) -> Result<String> {
13627    let out = tokio::process::Command::new("gh")
13628        .args([
13629            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13630        ])
13631        .current_dir(cwd)
13632        .quiet()
13633        .stdin(std::process::Stdio::null())
13634        .output()
13635        .await
13636        .context("spawn gh")?;
13637    if out.status.success() {
13638        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13639    } else {
13640        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13641    }
13642}
13643
13644/// Never force deletion of uncommitted work, including unknown legacy trees.
13645async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13646    match std::fs::symlink_metadata(path) {
13647        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13648            // A previous cleanup may have removed the tree before a branch
13649            // deletion or state save failed. Unregister it if still present.
13650            git::worktree_remove_clean(repo, path).await?;
13651            return Ok(());
13652        }
13653        Err(e) => return Err(e.into()),
13654        Ok(_) => {}
13655    }
13656    if !git::is_clean(path).await? {
13657        bail!(
13658            "worktree has uncommitted work; refusing removal: {}",
13659            path.display()
13660        );
13661    }
13662    if !git::worktree_remove_clean(repo, path).await? {
13663        bail!("worktree removal refused: {}", path.display());
13664    }
13665    Ok(())
13666}
13667
13668/// Read seats return replies, not working-tree changes. Only registered,
13669/// detached seats directly inside this run's root may discard those changes.
13670async fn remove_read_seat_worktree(state: &mut RunState, path: &Path) -> Result<()> {
13671    if !crate::retention::read_seat(path) || !path.exists() {
13672        return remove_preserved_worktree(&state.repo, path).await;
13673    }
13674    let owned = std::fs::symlink_metadata(path)?.is_dir()
13675        && path.parent().map(Path::canonicalize).transpose()?
13676            == Some(state.worktree_root().canonicalize()?)
13677        && crate::retention::worktrees(state)
13678            .await?
13679            .contains(&path.canonicalize()?);
13680    let common = crate::repo_lock::common_dir(&state.repo).await?;
13681    if !owned
13682        || common.is_none()
13683        || crate::repo_lock::common_dir(path).await? != common
13684        || git::git_raw(path, &["symbolic-ref", "-q", "HEAD"])
13685            .await?
13686            .ok()
13687        || state.candidates.iter().any(|c| c.worktree == path)
13688    {
13689        return remove_preserved_worktree(&state.repo, path).await;
13690    }
13691    let changes = git::status_porcelain(path).await?;
13692    if !git::worktree_remove(&state.repo, path).await? {
13693        bail!(
13694            "read-only seat worktree removal refused: {}",
13695            path.display()
13696        );
13697    }
13698    if !changes.is_empty() {
13699        let paths = changes.lines().take(20).collect::<Vec<_>>().join("; ");
13700        let paths: String = paths.chars().take(1024).collect();
13701        state.event(
13702            "fold",
13703            format!(
13704                "discarded read-only seat {} changes ({} status entries; first 20, at most 1024 chars): {paths}",
13705                path.file_name().unwrap_or_default().to_string_lossy(),
13706                changes.lines().count()
13707            ),
13708        );
13709    }
13710    Ok(())
13711}
13712
13713fn fold_target(
13714    home: &Path,
13715    root: &Path,
13716    path: &Path,
13717    isolated: bool,
13718) -> Option<crate::build_target::FoldTarget> {
13719    if !isolated {
13720        return None;
13721    }
13722    match crate::build_target::FoldTarget::for_checkout(home, path, root) {
13723        Ok(target) => target,
13724        Err(error) => {
13725            tracing::warn!(checkout = %path.display(), %error, "isolated target binding not proved at fold");
13726            None
13727        }
13728    }
13729}
13730
13731fn reclaim_fold_target(home: &Path, target: Option<crate::build_target::FoldTarget>) {
13732    if let Some(target) = target
13733        && let Err(error) = target.reclaim_orphaned_target(home)
13734    {
13735        tracing::warn!(%error, "isolated target retained after fold");
13736    }
13737}
13738
13739/// Tear a run's worktrees and branches down.
13740///
13741/// `home` is where the updated `run.json` is saved (via
13742/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13743/// a housekeeping pass already has its own honest `home` handed to it, and
13744/// falling through to the global here would write back through whichever
13745/// directory some other process or test pinned into that `OnceLock` first,
13746/// not the one the caller actually resolved its `runs` and `state` from.
13747pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13748    fold_run_with_policy(state, drop_winner, home, false).await
13749}
13750
13751/// Explicit CLI-only retention override; automatic callers use [`fold_run`].
13752pub async fn fold_run_overriding_recovery_retention(
13753    state: &mut RunState,
13754    drop_winner: bool,
13755    home: &Path,
13756) -> Result<Vec<String>> {
13757    fold_run_with_policy(state, drop_winner, home, true).await
13758}
13759
13760async fn fold_run_with_policy(
13761    state: &mut RunState,
13762    drop_winner: bool,
13763    home: &Path,
13764    override_recovery_retention: bool,
13765) -> Result<Vec<String>> {
13766    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state)
13767        && !override_recovery_retention
13768    {
13769        bail!("refusing fold: {reason}");
13770    }
13771    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13772        bail!("candidate preservation incomplete; refusing cleanup");
13773    }
13774    // Serialize ordinary folding with the same starts/resumes as target-only
13775    // eviction. Otherwise a second janitor could remove the whole worktree
13776    // around a live output borrower even though target eviction was fenced.
13777    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13778    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13779    // An override must use current durable ownership, not a pre-lock snapshot.
13780    if override_recovery_retention {
13781        *state = RunState::load_under(&state.id, home)?;
13782        anyhow::ensure!(
13783            state.status.done() && !state.status.resumable(),
13784            "recovery retention override requires a finished, non-resumable run"
13785        );
13786        let claimed = crate::daemon::is_working_on_other(home, &state.id, Timestamp::now())?;
13787        anyhow::ensure!(
13788            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
13789            "run is live or its ownership is unknown"
13790        );
13791        anyhow::ensure!(
13792            state.candidate_preservation.is_empty() && state.interrupted_implementations.is_empty(),
13793            "candidate preservation incomplete; refusing cleanup"
13794        );
13795    }
13796    let retention_reason = crate::pr_recovery::source_retention_reason(home, state);
13797    if let Some(reason) = &retention_reason
13798        && !override_recovery_retention
13799    {
13800        bail!("refusing fold: {reason}");
13801    }
13802    let _driver = crate::driver::maintenance(home, &state.id)?;
13803    let mut output_guards = Vec::new();
13804    for entry in std::fs::read_dir(state.worktree_root())
13805        .into_iter()
13806        .flatten()
13807        .flatten()
13808    {
13809        if entry.path().join(".git").is_file() {
13810            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13811        }
13812    }
13813    let repo = state.repo.clone();
13814    // CLI, Web and automatic cleanup all enter here, independently of
13815    // Runner::fold_losers. Protect deletion and shared config release too.
13816    let wait = repository_wait(state, None);
13817    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13818        Ok(lease) => lease,
13819        Err(e) => {
13820            state.save_under(home)?;
13821            return Err(e);
13822        }
13823    };
13824    // Confirmed merges dispose of their owned sources after durable evidence
13825    // capture. Preflight every source before removing anything.
13826    let _source_outputs = if retention_reason.is_some()
13827        || (state.status == RunStatus::Merged && state.merge.as_ref().is_some_and(|merge| merge.ok))
13828    {
13829        verify_merged_fold_sources(state, drop_winner, home).await?
13830    } else {
13831        Vec::new()
13832    };
13833    if let Some(reason) = retention_reason {
13834        state.event(
13835            "fold",
13836            format!(
13837                "operator CLI --override-recovery-retention authorized deletion for run {}: bypassed {reason}",
13838                state.id
13839            ),
13840        );
13841        state.save_under(home)?;
13842    }
13843    let root = state.worktree_root();
13844    let winner = state.tally.as_ref().map(|t| t.winner);
13845    let mut removed = Vec::new();
13846    let isolated = crate::build_target::isolated(&state.config);
13847
13848    for i in 0..state.candidates.len() {
13849        let c = state.candidates[i].clone();
13850        let is_winner = Some(c.label) == winner;
13851        if is_winner && !drop_winner {
13852            continue;
13853        }
13854        let existed = c.worktree.exists();
13855        let target = fold_target(home, &root, &c.worktree, isolated);
13856        // A vanished directory can still have a Git registration that keeps
13857        // its branch checked out. Remove that registration before the branch.
13858        remove_preserved_worktree(&repo, &c.worktree).await?;
13859        reclaim_fold_target(home, target);
13860        if existed {
13861            removed.push(c.worktree.to_string_lossy().into_owned());
13862        }
13863        // A branch handed to a later run (and its pull request) is not this
13864        // run's to delete.
13865        let handed_over = state.released_branches.contains(&c.branch);
13866        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13867            if !git::branch_delete(&repo, &c.branch).await? {
13868                bail!("candidate branch removal refused");
13869            }
13870            removed.push(c.branch.clone());
13871        }
13872        state.candidates[i].folded = true;
13873    }
13874
13875    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13876        let path = name.path();
13877        let keep = !drop_winner
13878            && winner.is_some_and(|w| {
13879                path.file_name()
13880                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13881            });
13882        if keep {
13883            continue;
13884        }
13885        let target = fold_target(home, &root, &path, isolated);
13886        remove_read_seat_worktree(state, &path).await?;
13887        reclaim_fold_target(home, target);
13888        removed.push(path.to_string_lossy().into_owned());
13889    }
13890
13891    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13892    // judge worktrees, so once the loop above has cleared all of them out,
13893    // the parent is a bare directory nobody else was ever going to remove -
13894    // git only ever managed what was inside it. Left alone, one of these
13895    // accumulates per fully-folded run; the operator's own machine had 74.
13896    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13897    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13898    // so does anything a run never claimed that happens to share the bay.
13899    remove_if_empty(&root);
13900
13901    if state.enabled_worktree_config && drop_winner {
13902        // A release, not a raw disable: some sibling run in this repository
13903        // may still hold its own reference (see `git::acquire_worktree_config`),
13904        // and only the last release actually turns the setting back off.
13905        git::release_worktree_config(&repo).await.ok();
13906        state.enabled_worktree_config = false;
13907    }
13908    state.save_under(home)?;
13909    Ok(removed)
13910}
13911
13912async fn verify_merged_fold_sources(
13913    state: &RunState,
13914    drop_winner: bool,
13915    home: &Path,
13916) -> Result<Vec<std::fs::File>> {
13917    let winner = state.tally.as_ref().map(|t| t.winner);
13918    let mut heads = BTreeSet::new();
13919    for candidate in &state.candidates {
13920        if Some(candidate.label) == winner && !drop_winner {
13921            continue;
13922        }
13923        anyhow::ensure!(
13924            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13925            "candidate recovery source is outside the inventoried run root: {}",
13926            candidate.worktree.display()
13927        );
13928        match std::fs::symlink_metadata(&candidate.worktree) {
13929            Ok(_) => {}
13930            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13931            Err(error) => return Err(error.into()),
13932        }
13933        if !state.released_branches.contains(&candidate.branch)
13934            && git::branch_exists(&state.repo, &candidate.branch).await?
13935        {
13936            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13937        }
13938    }
13939    let mut cargo_guards = Vec::new();
13940    let entries = match std::fs::read_dir(state.worktree_root()) {
13941        Ok(entries) => Some(entries),
13942        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13943        Err(error) => return Err(error.into()),
13944    };
13945    for entry in entries.into_iter().flatten() {
13946        let path = entry?.path();
13947        let metadata = std::fs::symlink_metadata(&path)?;
13948        anyhow::ensure!(
13949            metadata.is_dir() && !metadata.file_type().is_symlink(),
13950            "non-directory or symlink recovery source retained: {}",
13951            path.display()
13952        );
13953        if !drop_winner
13954            && winner.is_some_and(|w| {
13955                path.file_name()
13956                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13957            })
13958        {
13959            continue;
13960        }
13961        anyhow::ensure!(
13962            path.join(".git").is_file(),
13963            "unregistered recovery source retained: {}",
13964            path.display()
13965        );
13966        anyhow::ensure!(
13967            git::git(
13968                &path,
13969                &[
13970                    "status",
13971                    "--porcelain",
13972                    "--untracked-files=all",
13973                    "--ignored",
13974                    "--",
13975                    ".",
13976                    ":(exclude)target"
13977                ]
13978            )
13979            .await?
13980            .is_empty(),
13981            "source changes or ignored recovery files retained: {}",
13982            path.display()
13983        );
13984        match std::fs::symlink_metadata(path.join("target")) {
13985            Ok(_) => {
13986                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13987            }
13988            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13989            Err(error) => return Err(error.into()),
13990        }
13991        for checkpoint in [
13992            "rebase-merge",
13993            "rebase-apply",
13994            "sequencer",
13995            "CHERRY_PICK_HEAD",
13996            "MERGE_HEAD",
13997            "REBASE_HEAD",
13998            "REVERT_HEAD",
13999            "BISECT_LOG",
14000            "AUTO_MERGE",
14001            "MERGE_MSG",
14002        ] {
14003            let location = git::git(
14004                &path,
14005                &[
14006                    "rev-parse",
14007                    "--path-format=absolute",
14008                    "--git-path",
14009                    checkpoint,
14010                ],
14011            )
14012            .await?;
14013            anyhow::ensure!(
14014                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
14015                if e.kind() == std::io::ErrorKind::NotFound),
14016                "Git recovery checkpoint retained: {checkpoint}"
14017            );
14018        }
14019        heads.insert(git::rev_parse(&path, "HEAD").await?);
14020    }
14021    crate::fold_evidence::preserve(state, home, &heads).await?;
14022
14023    Ok(cargo_guards)
14024}
14025
14026/// Remove `dir` if it exists and has nothing in it.
14027///
14028/// Best-effort and silent by design: a directory that is not empty (a run
14029/// whose winner is still parked there, a stray file some other process left)
14030/// is exactly the case this must refuse, and a directory that is already gone
14031/// is not a failure worth reporting either. `std::fs::remove_dir` itself
14032/// already refuses a non-empty directory, so the emptiness check below is
14033/// belt, not suspenders - it is what keeps this from ever attempting the
14034/// removal in the case that matters, rather than trusting `remove_dir`'s
14035/// error path to have no side effects if it ever changed.
14036fn remove_if_empty(dir: &Path) {
14037    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
14038        std::fs::remove_dir(dir).ok();
14039    }
14040}
14041
14042/// Severity of the worst open finding in the last review round, for reporting.
14043pub fn worst_open(state: &RunState) -> Option<Severity> {
14044    state
14045        .reviews
14046        .last()?
14047        .reviews
14048        .iter()
14049        .flat_map(|r| r.findings.iter())
14050        .map(|f| f.severity)
14051        .max()
14052}
14053
14054/// `[worktree] setup` hides its products per worktree, which needs
14055/// `extensions.worktreeConfig`. Taken once per run and released with the run's
14056/// worktrees, like the hook's; every place that runs setup calls this first.
14057async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
14058    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
14059        git::acquire_worktree_config(repo).await?;
14060        state.enabled_worktree_config = true;
14061    }
14062    Ok(())
14063}
14064
14065#[cfg(test)]
14066mod tests {
14067    #[tokio::test]
14068    async fn launcher_base_fetch_requires_remote_success() {
14069        let tmp = tempfile::tempdir().unwrap();
14070        let repo = tmp.path().join("repo");
14071        let remote = tmp.path().join("remote");
14072        std::fs::create_dir(&repo).unwrap();
14073        std::fs::create_dir(&remote).unwrap();
14074        git::git(&remote, &["init", "--bare", "-b", "main"])
14075            .await
14076            .unwrap();
14077        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
14078        git::git(
14079            &repo,
14080            &[
14081                "-c",
14082                "user.name=Fixture",
14083                "-c",
14084                "user.email=fixture@example.invalid",
14085                "commit",
14086                "--allow-empty",
14087                "-m",
14088                "fixture",
14089            ],
14090        )
14091        .await
14092        .unwrap();
14093        git::git(
14094            &repo,
14095            &["remote", "add", "origin", remote.to_str().unwrap()],
14096        )
14097        .await
14098        .unwrap();
14099        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
14100        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14101        assert_eq!(
14102            resolve_base(&repo, "main", "origin", None)
14103                .await
14104                .unwrap()
14105                .commit,
14106            head
14107        );
14108        #[cfg(unix)]
14109        {
14110            let policy: crate::config::Coordination = toml::from_str(
14111                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
14112            ).unwrap();
14113            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
14114            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
14115            let held = crate::repo_lock::SeatLease::operation_with(
14116                &repo,
14117                Some(&policy),
14118                &crate::repo_lock::WaitPolicy::default(),
14119            )
14120            .await
14121            .unwrap();
14122            assert!(held.is_some());
14123            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
14124            tokio::pin!(fetch);
14125            assert!(
14126                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
14127                    .await
14128                    .is_err()
14129            );
14130            assert!(
14131                !repo.join(".git/FETCH_HEAD").exists(),
14132                "no fetch before admission"
14133            );
14134            drop(held);
14135            let fetched = fetch.await.unwrap();
14136            assert_eq!(fetched.commit, head);
14137            assert_eq!(fetched.remote, "origin");
14138            assert_eq!(fetched.branch, "main");
14139        }
14140        // A surviving tracking ref is not successful fetch evidence.
14141        git::git(
14142            &repo,
14143            &[
14144                "remote",
14145                "set-url",
14146                "origin",
14147                tmp.path().join("missing").to_str().unwrap(),
14148            ],
14149        )
14150        .await
14151        .unwrap();
14152        let error = resolve_base(&repo, "main", "origin", None)
14153            .await
14154            .unwrap_err()
14155            .to_string();
14156        assert!(error.contains("cannot read origin/main"));
14157        assert!(error.contains("refusing to branch off"));
14158    }
14159
14160    #[test]
14161    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
14162        let leaked = vec!["chore(deps): update a crate".to_owned()];
14163        let own = "fix(daemon): apply a chosen action";
14164        assert!(should_retitle("", own, &leaked));
14165        assert!(should_retitle(
14166            &format!("{REVIEW_PROMPT_OPENING} `x`"),
14167            own,
14168            &leaked
14169        ));
14170        assert!(should_retitle(
14171            "chore: land candidate A of run 1",
14172            own,
14173            &leaked
14174        ));
14175        assert!(should_retitle(
14176            "magi: candidate A (uncommitted work)",
14177            own,
14178            &leaked
14179        ));
14180        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
14181        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
14182        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
14183    }
14184
14185    #[test]
14186    fn pr_merge_plan_creates_adopts_or_stops() {
14187        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
14188        assert_eq!(
14189            pr_merge_plan(Ok(land::OpenPr::One {
14190                url: "u".into(),
14191                title: "t".into()
14192            })),
14193            PrPlan::Adopt {
14194                url: "u".into(),
14195                title: "t".into()
14196            }
14197        );
14198        let PrPlan::Stop(many) =
14199            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
14200        else {
14201            panic!("many must stop");
14202        };
14203        assert!(many.contains('a') && many.contains('b'));
14204        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
14205            panic!("a failed lookup must stop");
14206        };
14207        assert!(err.contains("bad token"));
14208    }
14209
14210    use super::*;
14211    use crate::run::GateStatus;
14212    use std::collections::BTreeMap;
14213    use std::time::Duration;
14214
14215    fn conductor() -> AgentSpec {
14216        AgentSpec {
14217            id: "conductor".to_owned(),
14218            kind: crate::config::AgentKind::Command,
14219            model: None,
14220            command: vec!["true".to_owned()],
14221            extra_args: Vec::new(),
14222            env: BTreeMap::new(),
14223            prompt_delivery: None,
14224            max_turn_secs: None,
14225        }
14226    }
14227
14228    fn spec(id: &str) -> AgentSpec {
14229        AgentSpec {
14230            id: id.to_owned(),
14231            kind: crate::config::AgentKind::Command,
14232            model: None,
14233            command: vec!["true".to_owned()],
14234            extra_args: Vec::new(),
14235            env: BTreeMap::new(),
14236            prompt_delivery: None,
14237            max_turn_secs: None,
14238        }
14239    }
14240
14241    fn ids(xs: &[&str]) -> BTreeSet<String> {
14242        xs.iter().map(|s| (*s).to_owned()).collect()
14243    }
14244
14245    #[test]
14246    fn pick_successor_skips_an_agent_another_seat_holds() {
14247        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
14248        let roster = [spec("a"), spec("b"), spec("c")];
14249        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14250        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14251    }
14252
14253    #[test]
14254    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
14255        // The other seat started on c but was handed to d; c is free again.
14256        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
14257        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14258        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14259        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
14260        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
14261    }
14262
14263    #[test]
14264    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
14265        let roster = [spec("a"), spec("b")];
14266        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
14267        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14268        let carried = ids(&["b"]);
14269        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
14270        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14271    }
14272
14273    #[test]
14274    fn pick_successor_returns_none_without_any_untried_successor() {
14275        let roster = [spec("a"), spec("b")];
14276        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
14277        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
14278        let carried = ids(&["a"]);
14279        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
14280    }
14281
14282    #[test]
14283    fn pick_successor_rescue_avoids_another_seats_occupant() {
14284        // b is a carried failure and free; a is held by the other seat.
14285        let roster = [spec("a"), spec("b"), spec("c")];
14286        let carried = ids(&["b", "c"]);
14287        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
14288        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14289    }
14290
14291    #[test]
14292    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
14293        let roster = [spec("a"), spec("b"), spec("c")];
14294        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
14295        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
14296    }
14297
14298    #[test]
14299    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
14300        let roster = [spec("a"), spec("b"), spec("c")];
14301        let failed = ids(&["a", "b", "c"]);
14302        // Rescue looks at the whole roster, once per id, then runs out.
14303        let mut tried = ids(&["b"]);
14304        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14305        assert_eq!(first.id, "a");
14306        tried.insert(first.id.clone());
14307        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
14308        assert_eq!(second.id, "c");
14309        tried.insert(second.id.clone());
14310        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
14311    }
14312
14313    #[test]
14314    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
14315        let roster = [spec("a"), spec("b")];
14316        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
14317        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14318    }
14319
14320    #[test]
14321    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
14322        let roster = [spec("a"), spec("b"), spec("c")];
14323        let h = SeatHistory {
14324            failed: ids(&["a"]),
14325            last_ok: Some("b".to_owned()),
14326            last_fail: None,
14327        };
14328        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
14329        // A last answerer that left the roster, or later failed, is ignored.
14330        let gone = SeatHistory {
14331            last_ok: Some("zzz".to_owned()),
14332            ..h.clone()
14333        };
14334        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
14335        let failed = SeatHistory {
14336            failed: ids(&["a", "b"]),
14337            last_ok: Some("b".to_owned()),
14338            last_fail: None,
14339        };
14340        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
14341    }
14342
14343    #[test]
14344    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
14345        let first = SeatState::new("review-1", "alpha", 7);
14346        let next = handover_seat("review-1", "gamma", 7);
14347        assert_ne!(first.claude_session, next.claude_session);
14348    }
14349
14350    #[test]
14351    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
14352        let mut state = state_with_summary("x", "y");
14353        let a = handover_seat("review-1", "beta", state.next_seat_seed());
14354        let b = handover_seat("review-1", "beta", state.next_seat_seed());
14355        assert_ne!(a.claude_session, b.claude_session);
14356    }
14357
14358    #[test]
14359    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
14360        let roster = [spec("a"), spec("b")];
14361        let h = SeatHistory {
14362            failed: ids(&["a", "b"]),
14363            ..SeatHistory::default()
14364        };
14365        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
14366        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
14367        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
14368    }
14369
14370    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
14371    // fallback loop depends on to terminate: it must walk forward from the
14372    // seat's own position, never restart at the front of the roster, and it
14373    // must never hand back an id already tried, however many times that id
14374    // happens to appear.
14375
14376    #[test]
14377    fn failure_signature_ignores_numbers_and_paths() {
14378        assert_eq!(
14379            failure_signature("exited with Some(2) and no usable output"),
14380            failure_signature("exited with Some(137) and no usable output")
14381        );
14382        assert_eq!(
14383            failure_signature("process exited with no usable output (exit code 2)"),
14384            failure_signature("process exited with no usable output (exit code 137)")
14385        );
14386        for (old, new) in [
14387            (
14388                "exited with Some(2) and no usable output",
14389                "process exited with no usable output (exit code 2)",
14390            ),
14391            (
14392                "exited with None and no usable output",
14393                "process exited with no usable output (exit code unavailable)",
14394            ),
14395        ] {
14396            let previous = FailClass::Other(failure_signature(old));
14397            let current = FailClass::Other(failure_signature(new));
14398            assert_eq!(previous, current, "resumption retains the failure class");
14399            assert!(!should_hand_over(Some(&previous), &current));
14400        }
14401        assert_eq!(
14402            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
14403            failure_signature("cannot open /var/x.txt: denied")
14404        );
14405        assert_ne!(failure_signature("boom"), failure_signature("bang"));
14406    }
14407
14408    #[test]
14409    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
14410        let other = FailClass::Other("x".into());
14411        assert!(should_hand_over(None, &FailClass::Quota));
14412        assert!(should_hand_over(Some(&other), &FailClass::Quota));
14413        assert!(should_hand_over(
14414            Some(&FailClass::Timeout),
14415            &FailClass::Timeout
14416        ));
14417        assert!(should_hand_over(None, &other));
14418        assert!(!should_hand_over(Some(&other), &other));
14419        assert!(should_hand_over(
14420            Some(&other),
14421            &FailClass::Other("y".into())
14422        ));
14423        // A quota or timeout in between ends the run of identical failures.
14424        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
14425        assert!(should_hand_over(Some(&FailClass::Quota), &other));
14426    }
14427
14428    #[test]
14429    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
14430        let a = SeatState::new("judge-1", "alpha", 7);
14431        let b = handover_seat("judge-1", "beta", 7);
14432        assert_ne!(a.claude_session, b.claude_session);
14433        assert_eq!(b.turns, 0);
14434    }
14435
14436    #[test]
14437    fn a_timeout_is_classified_apart_from_other_failures() {
14438        assert_eq!(
14439            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
14440            Some(FailClass::Timeout)
14441        );
14442        assert!(matches!(
14443            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
14444            Some(FailClass::Other(_))
14445        ));
14446    }
14447
14448    #[test]
14449    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
14450        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14451        let tried = BTreeSet::from(["beta".to_owned()]);
14452        // beta sits at index 1; the next candidate is gamma, never alpha —
14453        // which is very likely a different candidate slot's own agent.
14454        let next = next_untried_in_roster(&roster, 1, &tried);
14455        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
14456    }
14457
14458    #[test]
14459    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
14460        let roster = vec![spec("alpha"), spec("beta")];
14461        let tried = BTreeSet::from(["beta".to_owned()]);
14462        // beta is the roster's last entry: nothing follows it, and alpha —
14463        // earlier in the roster, almost certainly a different candidate
14464        // slot's own agent — must not be reached by wrapping back to it.
14465        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14466    }
14467
14468    #[test]
14469    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
14470        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
14471        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
14472        // beta (index 1) and gamma (index 2, the only entry after it) have
14473        // both been tried; alpha (index 0) never has, but it comes before
14474        // beta's own position, so there is nothing further for this seat.
14475        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
14476    }
14477
14478    #[test]
14479    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
14480        let roster = vec![spec("a"), spec("a"), spec("b")];
14481        let tried = BTreeSet::from(["a".to_owned()]);
14482        let next = next_untried_in_roster(&roster, 0, &tried);
14483        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
14484    }
14485
14486    #[test]
14487    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
14488        let roster = vec![spec("a"), spec("b")];
14489        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
14490        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
14491    }
14492
14493    #[test]
14494    fn remove_if_empty_only_ever_takes_a_bare_directory() {
14495        let dir = tempfile::tempdir().unwrap();
14496        let bay = dir.path().join("ffff");
14497
14498        // Not there yet: nothing to do, nothing to panic on.
14499        remove_if_empty(&bay);
14500        assert!(!bay.exists());
14501
14502        // Something still inside - the winner's worktree, or a stray file -
14503        // keeps the directory standing.
14504        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
14505        remove_if_empty(&bay);
14506        assert!(bay.exists(), "non-empty directory must survive");
14507
14508        // Once the last entry is gone, so is the directory itself.
14509        std::fs::remove_dir(bay.join("cand-A")).unwrap();
14510        remove_if_empty(&bay);
14511        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
14512    }
14513
14514    // `round_is_clean` is the exact decision this task fixed: a round with a
14515    // seat that never answered must not read the same as a round every seat
14516    // actually reviewed. These are deterministic and process-free by design —
14517    // the equivalent end-to-end check (a real reviewer timing out under a
14518    // live graph run) is a genuine race against wall-clock contention, and a
14519    // spawn slow enough to blow even a generous budget under a loaded test
14520    // run must not turn this specific regression check flaky.
14521
14522    #[test]
14523    fn a_full_panel_that_found_nothing_is_clean() {
14524        assert!(round_is_clean(
14525            0,
14526            true,
14527            2,
14528            2,
14529            0,
14530            IncompleteReviewPolicy::Block
14531        ));
14532    }
14533
14534    #[test]
14535    fn a_missing_seat_is_never_clean_under_the_default_policy() {
14536        assert!(!round_is_clean(
14537            0,
14538            true,
14539            1,
14540            2,
14541            0,
14542            IncompleteReviewPolicy::Block
14543        ));
14544    }
14545
14546    #[test]
14547    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
14548        assert!(!round_is_clean(
14549            1,
14550            true,
14551            1,
14552            2,
14553            0,
14554            IncompleteReviewPolicy::Warn
14555        ));
14556    }
14557
14558    #[test]
14559    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
14560        assert!(round_is_clean(
14561            0,
14562            true,
14563            1,
14564            2,
14565            0,
14566            IncompleteReviewPolicy::Warn
14567        ));
14568    }
14569
14570    #[test]
14571    fn a_full_panel_with_an_open_finding_is_not_clean() {
14572        assert!(!round_is_clean(
14573            1,
14574            true,
14575            2,
14576            2,
14577            0,
14578            IncompleteReviewPolicy::Block
14579        ));
14580    }
14581
14582    #[test]
14583    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14584        assert!(!round_is_clean(
14585            0,
14586            false,
14587            2,
14588            2,
14589            0,
14590            IncompleteReviewPolicy::Block
14591        ));
14592    }
14593
14594    // The stall this task closes: under the default `block` policy, a seat
14595    // missing only because it was rate limited must not force a wait for a
14596    // session limit that will not lift by the next round. `round_is_clean`
14597    // is where that quorum carve-out lives; the review loop around it never
14598    // changes what a reviewer's vote or a finding's severity means.
14599
14600    #[test]
14601    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14602        // 1 of 2 answered, and the one missing was quota'd — the exact
14603        // "review-2 rate limited (quota)" shape from the field report.
14604        assert!(round_is_clean(
14605            0,
14606            true,
14607            1,
14608            2,
14609            1,
14610            IncompleteReviewPolicy::Block
14611        ));
14612    }
14613
14614    #[test]
14615    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14616        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14617        // not a quota loss (`quota_missing` stays 0) — worth another try.
14618        assert!(!round_is_clean(
14619            0,
14620            true,
14621            1,
14622            2,
14623            0,
14624            IncompleteReviewPolicy::Block
14625        ));
14626    }
14627
14628    #[test]
14629    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14630        assert!(!round_is_clean(
14631            1,
14632            true,
14633            1,
14634            2,
14635            1,
14636            IncompleteReviewPolicy::Block
14637        ));
14638        assert!(!round_is_clean(
14639            0,
14640            false,
14641            1,
14642            2,
14643            1,
14644            IncompleteReviewPolicy::Block
14645        ));
14646    }
14647
14648    #[test]
14649    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14650        // Every seat quota'd, nobody answered: there is no panel to decide
14651        // on, so this must fall through to the existing block-and-retry
14652        // fallback rather than call an unreviewed patch clean.
14653        assert!(!round_is_clean(
14654            0,
14655            true,
14656            0,
14657            2,
14658            2,
14659            IncompleteReviewPolicy::Block
14660        ));
14661    }
14662
14663    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14664        CommandOutcome {
14665            command: "test".to_owned(),
14666            code,
14667            output_tail: String::new(),
14668            duration_ms: 0,
14669            resource_blocked,
14670        }
14671    }
14672
14673    #[cfg(unix)]
14674    #[tokio::test]
14675    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14676        ask_test_home();
14677        let temp = tempfile::tempdir().unwrap();
14678        let mut runner = runner_at(RunStatus::Gating);
14679        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14680        let commands = vec![
14681            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14682            "touch must-not-run".to_owned(),
14683        ];
14684        let (outcomes, retried, pids) = run_e2e_with_retry(
14685            &mut runner.state,
14686            &shell,
14687            &commands,
14688            temp.path(),
14689            Duration::from_secs(5),
14690            "fixture",
14691            None,
14692        )
14693        .await;
14694        assert_eq!(outcomes.len(), 1);
14695        assert!(outcomes[0].resource_blocked);
14696        assert!(!outcomes[0].build_failed());
14697        assert!(!retried);
14698        assert!(pids.is_empty());
14699        assert!(!temp.path().join("must-not-run").exists());
14700        assert_eq!(
14701            runner.state.resource_block.as_ref().unwrap().kind,
14702            "incremental cache admission"
14703        );
14704    }
14705
14706    #[test]
14707    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14708        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14709        assert!(
14710            !verify_inconclusive(&[outcome(Some(1), false)]),
14711            "an ordinary failure is still evidence about the patch"
14712        );
14713        assert!(verify_inconclusive(&[outcome(None, true)]));
14714        assert!(
14715            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14716            "one inconclusive outcome taints the whole batch"
14717        );
14718        assert!(!verify_inconclusive(&[]));
14719    }
14720
14721    #[tokio::test]
14722    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14723        // Alive for the first two checks, then dead - confirms the loop
14724        // actually re-polls rather than deciding once and sleeping out the
14725        // ceiling regardless.
14726        let calls = std::sync::atomic::AtomicUsize::new(0);
14727        let started = Instant::now();
14728        wait_for_pids_with(
14729            &[123],
14730            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14731            Duration::from_millis(5),
14732            Duration::from_secs(5),
14733        )
14734        .await;
14735        assert!(
14736            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14737            "must keep checking rather than deciding on the first answer"
14738        );
14739        assert!(
14740            started.elapsed() < Duration::from_secs(1),
14741            "must return the moment it is confirmed dead, not wait out the ceiling"
14742        );
14743    }
14744
14745    #[tokio::test]
14746    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14747        let started = Instant::now();
14748        wait_for_pids_with(
14749            &[123],
14750            |_| true, // never reports dead
14751            Duration::from_millis(5),
14752            Duration::from_millis(30),
14753        )
14754        .await;
14755        let elapsed = started.elapsed();
14756        assert!(
14757            elapsed >= Duration::from_millis(30),
14758            "must not give up before its own ceiling: {elapsed:?}"
14759        );
14760        assert!(
14761            elapsed < Duration::from_secs(1),
14762            "must not wait past its own ceiling either: {elapsed:?}"
14763        );
14764    }
14765
14766    #[tokio::test]
14767    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14768        let started = Instant::now();
14769        wait_for_pids_with(
14770            &[],
14771            |_| true,
14772            Duration::from_secs(5),
14773            Duration::from_secs(5),
14774        )
14775        .await;
14776        assert!(
14777            started.elapsed() < Duration::from_millis(200),
14778            "an empty pid list has nothing to confirm"
14779        );
14780    }
14781
14782    // `review_conclusion` is the exact decision the review hand-off task
14783    // fixed: a round budget spent (or a tree that stopped moving) must not
14784    // collapse into `Blocked` regardless of what verification actually
14785    // said. Deterministic and process-free for the same reason the
14786    // `round_is_clean` family above is.
14787    fn review_round(
14788        clean: bool,
14789        blocking: usize,
14790        answered: usize,
14791        expected: usize,
14792        progressed: bool,
14793        e2e_ok: bool,
14794    ) -> ReviewRound {
14795        ReviewRound {
14796            round: 1,
14797            head: "h".to_owned(),
14798            verified_head: None,
14799            verified_at: None,
14800            reviews: Vec::new(),
14801            e2e: vec![CommandOutcome {
14802                command: "test".to_owned(),
14803                code: Some(if e2e_ok { 0 } else { 1 }),
14804                output_tail: String::new(),
14805                duration_ms: 0,
14806                resource_blocked: false,
14807            }],
14808            verify_retried: false,
14809            e2e_deferred: false,
14810            e2e_defer_reason: None,
14811            fix: None,
14812            blocking,
14813            answered,
14814            expected,
14815            clean,
14816            progressed,
14817            vote_split: false,
14818            reconsideration: Vec::new(),
14819            verdict: None,
14820        }
14821    }
14822
14823    #[test]
14824    fn review_conclusion_is_none_when_nothing_has_run() {
14825        assert_eq!(review_conclusion(&[], 3), None);
14826    }
14827
14828    #[test]
14829    fn review_conclusion_is_none_while_rounds_remain() {
14830        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14831        assert_eq!(review_conclusion(&rounds, 3), None);
14832    }
14833
14834    #[test]
14835    fn review_conclusion_is_gating_once_a_round_is_clean() {
14836        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14837        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14838    }
14839
14840    #[test]
14841    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14842        let rounds = vec![
14843            review_round(false, 1, 2, 2, true, true),
14844            review_round(false, 1, 2, 2, true, true),
14845        ];
14846        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14847    }
14848
14849    #[test]
14850    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14851        let rounds = vec![
14852            review_round(false, 1, 2, 2, true, true),
14853            review_round(false, 1, 2, 2, true, false),
14854        ];
14855        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14856    }
14857
14858    #[test]
14859    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14860        // Magi never got a command to run against this round's own head — a
14861        // resource-blocked attempt, not a red one — so this must never
14862        // settle on `Blocked` the way a genuine e2e failure would. `None`
14863        // here is what tells `Runner::review_loop` to retry the check
14864        // itself rather than trust this cheap recomputation with a verdict
14865        // it cannot actually produce.
14866        let mut blocked = review_round(false, 1, 2, 2, true, false);
14867        blocked.e2e[0].resource_blocked = true;
14868        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14869        assert_eq!(review_conclusion(&rounds, 2), None);
14870    }
14871
14872    #[test]
14873    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14874        // Missing input, not a verified tree — never a hand-off candidate.
14875        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14876        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14877    }
14878
14879    #[test]
14880    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14881        let rounds = vec![
14882            review_round(false, 1, 2, 2, false, true),
14883            review_round(false, 1, 2, 2, false, true),
14884        ];
14885        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14886    }
14887
14888    fn secs(n: u64) -> Duration {
14889        Duration::from_secs(n)
14890    }
14891
14892    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14893    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14894    #[tokio::test]
14895    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14896        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14897        if std::env::var_os(CHILD).is_none() {
14898            // Other tests reconcile every run in the process-global home,
14899            // briefly taking owner locks. This ancestry test expects an idle
14900            // predecessor, so give it its own process and pinned test home.
14901            let output = std::process::Command::new(std::env::current_exe().unwrap())
14902                .args([
14903                    "--exact",
14904                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14905                    "--nocapture",
14906                ])
14907                .env(CHILD, "1")
14908                .env_remove("MAGI_WORKER_HOME")
14909                .env_remove("MAGI_WORKER_ID")
14910                .output()
14911                .unwrap();
14912            assert!(
14913                output.status.success()
14914                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14915                "{}\n{}",
14916                String::from_utf8_lossy(&output.stdout),
14917                String::from_utf8_lossy(&output.stderr)
14918            );
14919            return;
14920        }
14921        ask_test_home();
14922        let tmp = tempfile::tempdir().unwrap();
14923        let repo = tmp.path().join("repo");
14924        std::fs::create_dir_all(&repo).unwrap();
14925        init_repo(&repo);
14926        let g = |args: &[&str]| {
14927            let out = std::process::Command::new("git")
14928                .args(args)
14929                .current_dir(&repo)
14930                .quiet()
14931                .output()
14932                .unwrap();
14933            assert!(
14934                out.status.success(),
14935                "{}",
14936                String::from_utf8_lossy(&out.stderr)
14937            );
14938        };
14939        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14940        g(&["checkout", "-b", "historical"]);
14941        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14942        g(&["add", "."]);
14943        g(&["commit", "-m", "old alternative"]);
14944        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14945        g(&["checkout", "-b", "magi/task-selected", &base]);
14946        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14947        g(&["add", "."]);
14948        g(&["commit", "-m", "measurement infrastructure"]);
14949        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14950        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14951        g(&["add", "."]);
14952        g(&["commit", "-m", "dependent implementation"]);
14953        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14954        g(&["checkout", "main"]);
14955        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14956        g(&["add", "."]);
14957        g(&["commit", "-m", "unrelated base change"]);
14958        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14959        let origin = tmp.path().join("origin.git");
14960        g(&[
14961            "clone",
14962            "--bare",
14963            repo.to_str().unwrap(),
14964            origin.to_str().unwrap(),
14965        ]);
14966        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14967        g(&["fetch", "origin"]);
14968        let selection = crate::pr_recovery::Selection {
14969            url: "https://example.test/project/pull/1".into(),
14970            branch: "magi/task-selected".into(),
14971            head: head.clone(),
14972            source_head: None,
14973            source_run: None,
14974            source_proof: None,
14975            disposition: Some(crate::pr_recovery::Disposition::Review),
14976            instruction: "preserve measured behavior".into(),
14977            approved_at: jiff::Timestamp::now(),
14978        };
14979        let mut template = runner_at(RunStatus::Reviewing);
14980        template.state.repo = repo.clone();
14981        template.state.base_commit = current.clone();
14982        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14983        template.state.origin = Some(crate::run::Origin::queue("selected"));
14984        template.state.pr_head = Some(selection.branch.clone());
14985        template.state.pr_recovery = Some(selection.clone());
14986        let mut runner =
14987            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14988                .await
14989                .unwrap();
14990        let wt = runner.state.winner().unwrap().worktree.clone();
14991        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14992        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14993        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14994        assert!(runner.state.seeds.is_empty());
14995        assert_eq!(runner.state.instruction, "preserve measured behavior");
14996        runner.sync_to_base().await.unwrap();
14997        assert_eq!(
14998            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14999            "prerequisite"
15000        );
15001        assert_eq!(
15002            std::fs::read_to_string(wt.join("implementation")).unwrap(),
15003            "measured implementation"
15004        );
15005        assert_eq!(
15006            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
15007            "current main change"
15008        );
15009        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
15010        assert_eq!(
15011            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
15012            head
15013        );
15014
15015        // A failed selected-source setup stays inspectable, never resumable as a new competition.
15016        let mut partial = runner_at(RunStatus::Reviewing);
15017        partial.state.repo = repo.clone();
15018        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
15019        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
15020            run: Some("exit 1".into()),
15021            ..Default::default()
15022        }];
15023        let selection = runner.state.pr_recovery.clone().unwrap();
15024        partial.state.pr_recovery = Some(selection.clone());
15025        partial.state.pr_head = Some(selection.branch.clone());
15026        partial.state.origin = Some(crate::run::Origin::queue("selected"));
15027        let id = partial.state.id.clone();
15028        let error = match Runner::open_selected(
15029            &repo,
15030            partial.state,
15031            partial.roles,
15032            1,
15033            runner.state.base_commit.clone(),
15034            selection,
15035        )
15036        .await
15037        {
15038            Ok(_) => panic!("setup must fail"),
15039            Err(e) => e,
15040        };
15041        assert_eq!(
15042            error
15043                .downcast_ref::<crate::run::StartupFailure>()
15044                .unwrap()
15045                .run,
15046            id
15047        );
15048        let failed = RunState::load(&id).unwrap();
15049        assert_eq!(failed.status, RunStatus::Failed);
15050        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
15051        assert!(Runner::resume(&id).is_err());
15052        runner.state.status = RunStatus::Ready;
15053        runner.state.driver_exited = true;
15054        runner.state.save().unwrap();
15055        let mut retry = runner_at(RunStatus::Reviewing);
15056        retry.state.repo = repo.clone();
15057        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
15058        let mut reselected = runner.state.pr_recovery.clone().unwrap();
15059        reselected.approved_at = jiff::Timestamp::now();
15060        retry.state.origin = Some(crate::run::Origin::queue("selected"));
15061        retry.state.pr_head = Some(reselected.branch.clone());
15062        retry.state.pr_recovery = Some(reselected.clone());
15063        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
15064            .await
15065            .unwrap();
15066        let restarted = Runner::open_selected(
15067            &repo,
15068            retry.state,
15069            retry.roles,
15070            1,
15071            runner.state.base_commit.clone(),
15072            reselected,
15073        )
15074        .await
15075        .unwrap();
15076        assert_eq!(
15077            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
15078                .await
15079                .unwrap(),
15080            head
15081        );
15082        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
15083        assert_eq!(
15084            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
15085            head
15086        );
15087    }
15088
15089    #[tokio::test]
15090    #[cfg(unix)]
15091    async fn fallback_and_interrupted_restoration_require_coordination() {
15092        use std::os::fd::AsRawFd as _;
15093        ask_test_home();
15094        let tmp = tempfile::tempdir().unwrap();
15095        let mut runner = registered_runner(tmp.path()).await;
15096        runner.state.config.graph.timeout_resource_wait = 1;
15097        let winner = runner.state.candidates[0].clone();
15098        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15099        let leftover = winner.worktree.join("leftover");
15100        std::fs::write(&leftover, "untrusted edits").unwrap();
15101        let next = tmp.path().join("next-fixer");
15102        let mut failed = spec("first");
15103        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
15104        let mut successor = spec("second");
15105        successor.command = vec![
15106            "sh".into(),
15107            "-c".into(),
15108            format!("touch '{}'; echo done", next.display()),
15109        ];
15110        let attempts = vec![
15111            (failed.clone(), "first".into()),
15112            (successor.clone(), "second".into()),
15113        ];
15114        let lock_path = runner.state.repo.join(".git/updater.lock");
15115        let external = std::fs::File::create(&lock_path).unwrap();
15116        assert_eq!(
15117            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15118            0
15119        );
15120        let artifacts = tmp.path().join("artifacts");
15121        let build = |spec, seat| SeatJob {
15122            spec,
15123            seat,
15124            cwd: winner.worktree.clone(),
15125            prompt: "fixture".into(),
15126            timeout: Duration::from_secs(10),
15127            allow_write: true,
15128            sessions: false,
15129            artifacts: artifacts.clone(),
15130            stem: "fix".into(),
15131            handover: None,
15132        };
15133        let (_, _, out) = runner
15134            .ask_fixer_among(
15135                attempts.clone(),
15136                "graft",
15137                None,
15138                Some((&winner.worktree, &head)),
15139                build,
15140            )
15141            .await;
15142        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
15143        assert!(leftover.exists(), "contention must prevent clean");
15144        assert!(!next.exists());
15145        assert!(runner.state.handovers.is_empty());
15146        // Early interrupted-repair rollback uses the same protected route.
15147        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15148            from_run: "earlier".into(),
15149            generation: 1,
15150            mode: crate::recovery::Mode::VerifyRepair,
15151            reasons: Vec::new(),
15152            reviewed_head: None,
15153            rounds: Vec::new(),
15154            open_findings: Vec::new(),
15155            evidence: None,
15156            criteria_inherited: true,
15157            repair: crate::recovery::Repair::Started {
15158                before: head.clone(),
15159            },
15160        });
15161        assert!(
15162            runner
15163                .recovery_repair(&winner)
15164                .await
15165                .unwrap_err()
15166                .is::<crate::repo_lock::ResourceBlocked>()
15167        );
15168        assert!(matches!(
15169            runner.state.recovery.as_ref().unwrap().repair,
15170            crate::recovery::Repair::Started { .. }
15171        ));
15172        assert!(leftover.exists());
15173        drop(external);
15174        runner.recovery_repair(&winner).await.unwrap();
15175        assert!(!leftover.exists());
15176        assert!(matches!(
15177            runner.state.recovery.as_ref().unwrap().repair,
15178            crate::recovery::Repair::Done { .. }
15179        ));
15180        // A hard lock-open failure also leaves HEAD and untracked work intact.
15181        std::fs::remove_file(&lock_path).unwrap();
15182        std::fs::create_dir(&lock_path).unwrap();
15183        std::fs::write(&leftover, "untrusted edits").unwrap();
15184        let (_, _, out) = runner
15185            .ask_fixer_among(
15186                attempts,
15187                "graft",
15188                None,
15189                Some((&winner.worktree, &head)),
15190                build,
15191            )
15192            .await;
15193        assert!(matches!(out, AgentOutcome::Failed(_)));
15194        assert!(leftover.exists());
15195        assert!(!next.exists());
15196        assert_eq!(
15197            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15198            head
15199        );
15200    }
15201
15202    #[cfg(unix)]
15203    async fn registered_runner(root: &Path) -> Runner {
15204        repository_runner(root, crate::config::test_coordination()).await
15205    }
15206
15207    async fn repository_runner(
15208        root: &Path,
15209        coordination: Option<crate::config::Coordination>,
15210    ) -> Runner {
15211        ask_test_home();
15212        let repo = root.join("repo");
15213        std::fs::create_dir_all(&repo).unwrap();
15214        init_repo(&repo);
15215        if coordination.is_some() {
15216            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15217        }
15218        let mut runner = runner_at(RunStatus::Prep);
15219        runner.state.repo = repo.clone();
15220        runner.state.config.coordination = coordination;
15221        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15222        runner.state.config.graph.worktree_root = Some(root.join("wt"));
15223        runner.state.config.graph.advise = false;
15224        runner.roles.implementers = vec![spec("implement")];
15225        runner.roles.judges = vec![spec("judge")];
15226        runner.prep().await.unwrap();
15227        runner
15228    }
15229
15230    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
15231
15232    #[tokio::test]
15233    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
15234        let tmp = tempfile::tempdir().unwrap();
15235        let mut runner = repository_runner(tmp.path(), None).await;
15236        let prompts = runner.state.config.prompts.clone();
15237        let run = runner.state.id.clone();
15238        let ctx = WaveCtx {
15239            carry_seats: false,
15240            pauses: None,
15241            run: &run,
15242            node: "advise",
15243            prompts: &prompts,
15244            cache: None,
15245            round: None,
15246        };
15247        let jobs: Vec<_> = ["alpha", "beta"]
15248            .into_iter()
15249            .map(|id| SeatJob {
15250                spec: scripted(id, AUTH_OUTAGE),
15251                seat: SeatState::new(id, id, 7),
15252                prompt: "propose".into(),
15253                cwd: runner.state.candidates[0].worktree.clone(),
15254                timeout: Duration::from_secs(10),
15255                allow_write: false,
15256                sessions: false,
15257                artifacts: agent::artifacts_dir(&runner.state.dir()),
15258                stem: id.into(),
15259                handover: None,
15260            })
15261            .collect();
15262        let mut losses = Vec::new();
15263        let results = ask_wave_with(
15264            jobs.clone(),
15265            runner.sem.clone(),
15266            2,
15267            &[],
15268            &ctx,
15269            &mut losses,
15270            &mut runner.state,
15271            &|_: &str| Ok(()),
15272        )
15273        .await;
15274        assert!(
15275            results
15276                .iter()
15277                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
15278        );
15279        assert_eq!(runner.state.quota.len(), 2);
15280        assert!(
15281            runner
15282                .state
15283                .quota
15284                .iter()
15285                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
15286        );
15287        assert!(!runner.state.advise_attempted);
15288        runner.stop_for_resource().unwrap();
15289        assert_eq!(runner.state.status, RunStatus::Blocked);
15290        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
15291        assert_eq!(
15292            runner.state.resource_block.as_ref().unwrap().kind,
15293            "agent authentication"
15294        );
15295        // A machine loss followed by an ordinary failure is not evidence
15296        // that the whole node was unavailable, and must not earn a refund.
15297        runner.state.resource_block = None;
15298        runner.state.quota.clear();
15299        let mut mixed = jobs;
15300        mixed[1].spec.command = vec!["false".into()];
15301        let results = ask_wave_with(
15302            mixed,
15303            runner.sem.clone(),
15304            0,
15305            &[],
15306            &ctx,
15307            &mut losses,
15308            &mut runner.state,
15309            &|_: &str| Ok(()),
15310        )
15311        .await;
15312        assert!(results.iter().all(|(_, res, _)| res.is_err()));
15313        assert!(runner.state.resource_block.is_none());
15314        assert_eq!(losses.len(), 1);
15315    }
15316
15317    #[tokio::test]
15318    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
15319        let tmp = tempfile::tempdir().unwrap();
15320        let mut runner = repository_runner(tmp.path(), None).await;
15321        let implementer = scripted("implement", AUTH_OUTAGE);
15322        runner.roles.implementers = vec![implementer.clone()];
15323        runner.state.config.agents = vec![implementer];
15324        runner.implement().await.unwrap();
15325        assert_eq!(runner.state.status, RunStatus::Blocked);
15326        assert_eq!(runner.state.quota.len(), 1);
15327        assert_eq!(
15328            runner.state.quota[0].cause,
15329            agent::QuotaCause::Authentication
15330        );
15331        let candidate = &runner.state.candidates[0];
15332        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
15333        assert_eq!(runner.state.seats["impl-A"].turns, 1);
15334    }
15335
15336    #[tokio::test]
15337    #[cfg(unix)]
15338    async fn parked_implementer_persists_completed_work() {
15339        for registered in [false, true] {
15340            let tmp = tempfile::tempdir().unwrap();
15341            let mut runner = registered_runner(tmp.path()).await;
15342            if !registered {
15343                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
15344            }
15345            let marker = tmp.path().join("started");
15346            let release = tmp.path().join("release");
15347            let mut implementer = spec("implement");
15348            implementer.command = vec![
15349                "sh".into(),
15350                "-c".into(),
15351                format!(
15352                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
15353                    marker.display(),
15354                    release.display()
15355                ),
15356            ];
15357            runner.roles.implementers = vec![implementer.clone()];
15358            runner.state.config.agents = vec![implementer];
15359            let pause = runner.pause.clone();
15360            let (result, ()) = tokio::join!(runner.implement(), async {
15361                tokio::time::timeout(Duration::from_secs(10), async {
15362                    while !marker.exists() {
15363                        tokio::time::sleep(Duration::from_millis(10)).await;
15364                    }
15365                })
15366                .await
15367                .unwrap();
15368                pause.park();
15369                std::fs::write(&release, "go").unwrap();
15370            });
15371            result.unwrap();
15372            let candidate = &runner.state.candidates[0];
15373            assert!(candidate.commits > 0, "{candidate:?}");
15374            assert!(candidate.summary.contains("saved-result"));
15375            assert_eq!(candidate.files, 1);
15376            let before = candidate.commits;
15377            runner.implement().await.unwrap();
15378            assert_eq!(runner.state.candidates[0].commits, before);
15379        }
15380    }
15381
15382    #[tokio::test]
15383    #[cfg(unix)]
15384    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
15385        let tmp = tempfile::tempdir().unwrap();
15386        let mut runner = registered_runner(tmp.path()).await;
15387        let winner = runner.state.candidates[0].clone();
15388        let cache = tmp.path().join("cache");
15389        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
15390        runner.state.config.graph.timeout_resource_wait = 0;
15391        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
15392        let crate::cache::AcquireOutcome::Acquired(held) =
15393            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
15394        else {
15395            panic!("cache must be free")
15396        };
15397        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15398            from_run: "earlier".into(),
15399            generation: 1,
15400            mode: crate::recovery::Mode::VerifyRepair,
15401            reasons: Vec::new(),
15402            reviewed_head: None,
15403            rounds: Vec::new(),
15404            open_findings: Vec::new(),
15405            evidence: None,
15406            criteria_inherited: true,
15407            repair: crate::recovery::Repair::NotStarted,
15408        });
15409        runner.recovery_repair(&winner).await.unwrap();
15410        assert!(matches!(
15411            runner.state.recovery.as_ref().unwrap().repair,
15412            crate::recovery::Repair::WaitingForRestore { .. }
15413        ));
15414        drop(held);
15415        runner.state.resource_block = None;
15416        runner.state.config.graph.timeout_resource_wait = 1;
15417        runner.recovery_repair(&winner).await.unwrap();
15418        assert!(matches!(
15419            runner.state.recovery.as_ref().unwrap().repair,
15420            crate::recovery::Repair::Done { .. }
15421        ));
15422        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
15423    }
15424
15425    #[tokio::test]
15426    #[cfg(unix)]
15427    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
15428        use std::os::fd::AsRawFd as _;
15429        for graft in [false, true] {
15430            let tmp = tempfile::tempdir().unwrap();
15431            let mut runner = registered_runner(tmp.path()).await;
15432            runner.state.config.graph.timeout_resource_wait = 1;
15433            let winner = runner.state.candidates[0].clone();
15434            let marker = tmp.path().join("fixer-started");
15435            let mut fixer = spec("fixer");
15436            fixer.command = vec![
15437                "sh".into(),
15438                "-c".into(),
15439                format!("echo started >> '{}'; echo done", marker.display()),
15440            ];
15441            runner.roles.fixer = Some(vec![fixer]);
15442            if graft {
15443                let mut loser = winner.clone();
15444                loser.label = 'B';
15445                loser.index = 1;
15446                runner.state.candidates.push(loser);
15447                runner.state.config.graph.graft_runner_up = true;
15448                runner.state.tally = Some(Tally {
15449                    first_choice: BTreeMap::new(),
15450                    borda: BTreeMap::new(),
15451                    winner: winner.label,
15452                    rankings: 1,
15453                    unanimous_initial: true,
15454                    deliberated: false,
15455                    changed_votes: 0,
15456                    unanimous_final: true,
15457                    tie_break: None,
15458                    judges: 1,
15459                    present: 1,
15460                    quorum: 1,
15461                    met_quorum: true,
15462                    uncontested: None,
15463                });
15464                runner.state.judgements.push(crate::run::Judgement {
15465                    judge: 1,
15466                    seat: "judge-1".into(),
15467                    agent: "judge".into(),
15468                    ranking: vec![winner.label, 'B'],
15469                    reasons: BTreeMap::new(),
15470                    confidence: None,
15471                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
15472                    order: vec![0, 1],
15473                    failed: None,
15474                    duration_ms: 0,
15475                });
15476            } else {
15477                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
15478                    from_run: "earlier".into(),
15479                    generation: 1,
15480                    mode: crate::recovery::Mode::VerifyRepair,
15481                    reasons: Vec::new(),
15482                    reviewed_head: None,
15483                    rounds: Vec::new(),
15484                    open_findings: Vec::new(),
15485                    evidence: None,
15486                    criteria_inherited: true,
15487                    repair: crate::recovery::Repair::NotStarted,
15488                });
15489            }
15490            let reader = crate::repo_lock::SeatLease::for_checkout(
15491                &winner.worktree,
15492                false,
15493                crate::config::test_coordination().as_ref(),
15494            )
15495            .await
15496            .unwrap();
15497            let pause = runner.pause.clone();
15498            let (result, ()) = tokio::join!(
15499                async {
15500                    if graft {
15501                        runner.graft().await
15502                    } else {
15503                        runner.recovery_repair(&winner).await
15504                    }
15505                },
15506                async {
15507                    tokio::time::sleep(Duration::from_millis(100)).await;
15508                    pause.park();
15509                }
15510            );
15511            result.unwrap();
15512            assert!(!marker.exists(), "a blocked seat must not spawn");
15513            // Exercise the persisted state, as a binary replacement/resume does.
15514            runner.state = RunState::load(&runner.state.id).unwrap();
15515            if graft {
15516                assert_eq!(
15517                    runner.state.graft.as_ref().unwrap().outcome,
15518                    GraftOutcome::WaitingForRestore
15519                );
15520            } else {
15521                assert!(matches!(
15522                    runner.state.recovery.as_ref().unwrap().repair,
15523                    crate::recovery::Repair::WaitingForRestore { .. }
15524                ));
15525            }
15526            drop(reader);
15527            runner.pause = Pause::new();
15528            runner.state.resource_block = None;
15529            // Deferred restoration must still respect an external lock owner.
15530            let external =
15531                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15532            assert_eq!(
15533                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15534                0
15535            );
15536            let leftover = winner.worktree.join("leftover");
15537            std::fs::write(&leftover, "earlier fallback edits").unwrap();
15538            let result = if graft {
15539                runner.graft().await
15540            } else {
15541                runner.recovery_repair(&winner).await
15542            };
15543            assert!(
15544                result
15545                    .unwrap_err()
15546                    .is::<crate::repo_lock::ResourceBlocked>()
15547            );
15548            assert!(leftover.exists());
15549            assert!(!marker.exists());
15550            drop(external);
15551            runner.state.resource_block = None;
15552            let result = if graft {
15553                runner.graft().await
15554            } else {
15555                runner.recovery_repair(&winner).await
15556            };
15557            result.unwrap();
15558            assert!(!leftover.exists(), "restore must precede retry");
15559            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
15560            if graft {
15561                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15562            } else {
15563                assert!(matches!(
15564                    runner.state.recovery.as_ref().unwrap().repair,
15565                    crate::recovery::Repair::Done { .. }
15566                ));
15567            }
15568        }
15569    }
15570
15571    #[tokio::test]
15572    #[cfg(unix)]
15573    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15574        use std::os::fd::AsRawFd as _;
15575        let tmp = tempfile::tempdir().unwrap();
15576        let mut runner = registered_runner(tmp.path()).await;
15577        runner.state.config.graph.timeout_resource_wait = 1;
15578        let winner = runner.state.candidates[0].clone();
15579        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15580        runner.state.tally = Some(Tally {
15581            first_choice: BTreeMap::new(),
15582            borda: BTreeMap::new(),
15583            winner: winner.label,
15584            rankings: 0,
15585            unanimous_initial: false,
15586            deliberated: false,
15587            changed_votes: 0,
15588            unanimous_final: false,
15589            tie_break: None,
15590            judges: 0,
15591            present: 0,
15592            quorum: 0,
15593            met_quorum: true,
15594            uncontested: Some("fixture".into()),
15595        });
15596        runner.state.graft = Some(GraftRecord {
15597            pre: pre.clone(),
15598            post: None,
15599            lines: None,
15600            limit: None,
15601            agent: None,
15602            outcome: GraftOutcome::Running,
15603            requested: Vec::new(),
15604            adoption: None,
15605            adoption_error: None,
15606        });
15607        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15608        git::commit_all(&winner.worktree, "unfinished graft")
15609            .await
15610            .unwrap();
15611        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15612        let leftover = winner.worktree.join("leftover");
15613        std::fs::write(&leftover, "untrusted edits").unwrap();
15614        let path = runner.state.repo.join(".git/updater.lock");
15615        let external = std::fs::File::create(&path).unwrap();
15616        assert_eq!(
15617            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15618            0
15619        );
15620        assert!(
15621            runner
15622                .graft()
15623                .await
15624                .unwrap_err()
15625                .is::<crate::repo_lock::ResourceBlocked>()
15626        );
15627        assert!(matches!(
15628            runner.state.graft.as_ref().unwrap().outcome,
15629            GraftOutcome::Running
15630        ));
15631        assert_eq!(
15632            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15633            unfinished
15634        );
15635        assert!(leftover.exists());
15636        drop(external);
15637        std::fs::remove_file(&path).unwrap();
15638        std::fs::create_dir(&path).unwrap();
15639        assert!(runner.graft().await.is_err());
15640        assert!(matches!(
15641            runner.state.graft.as_ref().unwrap().outcome,
15642            GraftOutcome::Running
15643        ));
15644        assert_eq!(
15645            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15646            unfinished
15647        );
15648        assert!(leftover.exists());
15649        std::fs::remove_dir(&path).unwrap();
15650        runner.graft().await.unwrap();
15651        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15652        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15653        assert!(!leftover.exists());
15654    }
15655
15656    #[tokio::test]
15657    #[cfg(unix)]
15658    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15659        use std::os::fd::AsRawFd as _;
15660        let tmp = tempfile::tempdir().unwrap();
15661        let mut runner = registered_runner(tmp.path()).await;
15662        runner.state.prep_complete = Some(false);
15663        runner.state.config.graph.timeout_resource_wait = 1;
15664        runner.state.save().unwrap();
15665        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15666        assert_eq!(
15667            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15668            0
15669        );
15670        runner.execute().await.unwrap();
15671        let disk = RunState::load(&runner.state.id).unwrap();
15672        assert!(disk.driver_exited);
15673        assert_eq!(disk.driver_pid, None);
15674        assert_eq!(runner.state.status, RunStatus::Blocked);
15675        assert_eq!(
15676            runner.state.resource_block.as_ref().unwrap().kind,
15677            "repository coordination"
15678        );
15679        assert!(runner.state.prep_failure.is_none());
15680        assert!(runner.state.seats.is_empty());
15681        runner.pause.park();
15682        // Include headroom for Git subprocesses under parallel test load;
15683        // the repository contention deadline itself remains one second.
15684        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15685            .await
15686            .unwrap()
15687            .unwrap();
15688        assert!(runner.state.parked);
15689        assert!(runner.state.prep_failure.is_none());
15690        let disk = RunState::load(&runner.state.id).unwrap();
15691        assert!(disk.driver_exited);
15692        assert_eq!(disk.driver_pid, None);
15693    }
15694
15695    #[tokio::test]
15696    #[cfg(unix)]
15697    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15698        use std::os::fd::AsRawFd as _;
15699        let tmp = tempfile::tempdir().unwrap();
15700        let mut runner = registered_runner(tmp.path()).await;
15701        runner.state.prep_complete = Some(false);
15702        runner.state.config.graph.timeout_resource_wait = 30;
15703        runner.state.save().unwrap();
15704        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15705        assert_eq!(
15706            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15707            0
15708        );
15709        assert!(
15710            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15711                .await
15712                .is_err()
15713        );
15714        let disk = RunState::load(&runner.state.id).unwrap();
15715        assert!(disk.driver_exited);
15716        assert_eq!(disk.driver_pid, None);
15717        assert!(disk.status.resumable());
15718        assert!(runner.state.driver_exited);
15719        assert_eq!(runner.state.driver_pid, None);
15720        runner.state.save().unwrap();
15721        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15722    }
15723
15724    #[tokio::test]
15725    #[cfg(unix)]
15726    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15727        let tmp = tempfile::tempdir().unwrap();
15728        let mut runner = registered_runner(tmp.path()).await;
15729        let winner = runner.state.candidates[0].clone();
15730        let cache = tmp.path().join("cache");
15731        let started = tmp.path().join("pre-gate-started");
15732        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15733        runner.state.config.verify.gate =
15734            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15735        let home = crate::run::home();
15736        let owner =
15737            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15738        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15739            crate::cache::AcquireOutcome::Acquired(g) => g,
15740            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15741        };
15742        let mut gate = Box::pin(runner.run_gate(&winner));
15743        assert!(
15744            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15745                .await
15746                .is_err()
15747        );
15748        assert!(!started.exists(), "pre-gate must wait for the cache too");
15749        // Simulate the wave that already owns the cache entering invoke.
15750        // It must obtain the editor reservation and finish without waiting
15751        // for the gate that is queued behind its cache.
15752        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15753        let editor = crate::repo_lock::SeatLease::checkout_with(
15754            &winner.worktree,
15755            true,
15756            crate::config::test_coordination().as_ref(),
15757            &wait,
15758        )
15759        .await
15760        .unwrap();
15761        drop(editor);
15762        guard.release();
15763        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15764            .await
15765            .unwrap()
15766            .unwrap();
15767        drop(gate);
15768        assert!(outcomes.iter().all(CommandOutcome::ok));
15769        assert!(started.exists());
15770    }
15771
15772    #[tokio::test]
15773    #[cfg(unix)]
15774    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15775        use std::os::fd::AsRawFd as _;
15776
15777        ask_test_home();
15778        let tmp = tempfile::tempdir().unwrap();
15779        let repo = tmp.path().join("repo");
15780        std::fs::create_dir_all(&repo).unwrap();
15781        init_repo(&repo);
15782        let common = repo.join(".git");
15783        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15784        let mut runner = runner_at(RunStatus::Prep);
15785        runner.state.repo = repo.clone();
15786        runner.state.config.coordination = crate::config::test_coordination();
15787        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15788        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15789        runner.state.config.graph.advise = false;
15790        runner.roles.implementers = vec![spec("implement")];
15791        runner.roles.judges = vec![spec("judge")];
15792        runner.prep().await.unwrap();
15793        let winner = runner.state.candidates[0].clone();
15794        // Emulate edits already left by a completed seat. The fixture CLI
15795        // only reports completion; it performs no repository operations.
15796        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15797        let completed = tmp.path().join("seat-completed");
15798        runner.roles.implementers[0].command = vec![
15799            "sh".into(),
15800            "-c".into(),
15801            format!("touch '{}'; echo completed", completed.display()),
15802        ];
15803        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15804        assert_eq!(
15805            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15806            0
15807        );
15808        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15809        let mut implement = Box::pin(runner.implement());
15810        tokio::select! {
15811            result = &mut implement => panic!("rescue must wait: {result:?}"),
15812            result = tokio::time::timeout(Duration::from_secs(10), async {
15813                while !completed.exists() {
15814                    tokio::time::sleep(Duration::from_millis(20)).await;
15815                }
15816            }) => result.unwrap(),
15817        }
15818        assert!(
15819            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15820                .await
15821                .is_err()
15822        );
15823        assert_eq!(
15824            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15825            before
15826        );
15827        assert_eq!(
15828            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15829            0
15830        );
15831        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15832            .await
15833            .unwrap()
15834            .unwrap();
15835        drop(implement);
15836        assert_ne!(
15837            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15838            before
15839        );
15840        assert!(git::is_clean(&winner.worktree).await.unwrap());
15841
15842        let started = tmp.path().join("pre-gate-started");
15843        let proceed = tmp.path().join("pre-gate-proceed");
15844        let checked = tmp.path().join("gate-ran");
15845        runner.state.config.verify.pre_gate = vec![format!(
15846            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15847            started.display(),
15848            proceed.display()
15849        )];
15850        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15851        assert_eq!(
15852            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15853            0
15854        );
15855        let mut gate = Box::pin(runner.run_gate(&winner));
15856        assert!(
15857            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15858                .await
15859                .is_err()
15860        );
15861        assert!(!started.exists());
15862        assert!(!checked.exists());
15863        assert!(git::is_clean(&winner.worktree).await.unwrap());
15864        assert_eq!(
15865            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15866            0
15867        );
15868        tokio::select! {
15869            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15870            result = tokio::time::timeout(Duration::from_secs(10), async {
15871                while !started.exists() {
15872                    tokio::time::sleep(Duration::from_millis(20)).await;
15873                }
15874            }) => result.unwrap(),
15875        }
15876        assert_eq!(
15877            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15878            -1,
15879            "mechanical edits must hold the real lock"
15880        );
15881        std::fs::write(proceed, "ready").unwrap();
15882        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15883            .await
15884            .unwrap()
15885            .unwrap();
15886        drop(gate);
15887        assert!(outcomes.iter().all(CommandOutcome::ok));
15888        assert!(checked.exists());
15889        assert!(runner.state.pre_gate_commit.is_some());
15890        assert!(git::is_clean(&winner.worktree).await.unwrap());
15891        assert_eq!(
15892            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15893            0,
15894            "gate completion must release coordination"
15895        );
15896        drop(outsider);
15897        std::fs::remove_file(common.join("updater.lock")).unwrap();
15898        std::fs::create_dir(common.join("updater.lock")).unwrap();
15899        std::fs::remove_file(&started).unwrap();
15900        std::fs::remove_file(&checked).unwrap();
15901        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15902        let error = runner.run_gate(&winner).await.unwrap_err();
15903        assert!(
15904            error
15905                .to_string()
15906                .contains("acquire mandatory repository lock")
15907        );
15908        assert!(
15909            !started.exists(),
15910            "failed acquisition must prevent pre-gate"
15911        );
15912        assert!(!checked.exists(), "failed acquisition must prevent gate");
15913        assert!(git::is_clean(&winner.worktree).await.unwrap());
15914        assert_eq!(
15915            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15916            head
15917        );
15918    }
15919
15920    #[tokio::test]
15921    #[cfg(unix)]
15922    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15923        use std::os::fd::AsRawFd as _;
15924        use std::time::Duration;
15925
15926        ask_test_home();
15927        let tmp = tempfile::tempdir().unwrap();
15928        let repo = tmp.path().join("repo");
15929        std::fs::create_dir_all(&repo).unwrap();
15930        init_repo(&repo);
15931        let common = repo.join(".git");
15932        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15933        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15934        assert_eq!(
15935            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15936            0
15937        );
15938        let mut runner = runner_at(RunStatus::Prep);
15939        runner.state.repo = repo.clone();
15940        runner.state.config.coordination = crate::config::test_coordination();
15941        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15942        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15943        runner.state.config.graph.advise = true;
15944        runner.state.config.graph.advisors = 1;
15945        runner.roles.implementers = vec![spec("implement")];
15946        runner.roles.judges = vec![spec("judge")];
15947        let root = runner.state.worktree_root();
15948        let started = tmp.path().join("setup-started");
15949        let proceed = tmp.path().join("setup-proceed");
15950        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15951            run: Some(format!(
15952                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15953                started.display(),
15954                proceed.display()
15955            )),
15956            ..Default::default()
15957        }];
15958        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15959        let config = git::git(&repo, &["config", "--local", "--list"])
15960            .await
15961            .unwrap();
15962        let mut prep = Box::pin(runner.prep());
15963        assert!(
15964            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15965                .await
15966                .is_err()
15967        );
15968        assert!(
15969            !root.exists(),
15970            "contention must prevent even checkout creation"
15971        );
15972        assert!(!started.exists(), "contention must prevent setup");
15973        assert_eq!(
15974            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15975            branches
15976        );
15977        assert_eq!(
15978            git::git(&repo, &["config", "--local", "--list"])
15979                .await
15980                .unwrap(),
15981            config
15982        );
15983        drop(lock);
15984        tokio::select! {
15985            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15986            result = tokio::time::timeout(Duration::from_secs(10), async {
15987                while !started.exists() {
15988                    tokio::time::sleep(Duration::from_millis(20)).await;
15989                }
15990            }) => result.unwrap(),
15991        }
15992        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15993        assert_eq!(
15994            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15995            -1
15996        );
15997        assert!(
15998            tokio::time::timeout(
15999                Duration::from_millis(100),
16000                crate::repo_lock::SeatLease::for_checkout(
16001                    &repo,
16002                    true,
16003                    crate::config::test_coordination().as_ref()
16004                )
16005            )
16006            .await
16007            .is_err(),
16008            "preparation must also exclude local editors"
16009        );
16010        std::fs::write(proceed, "ready").unwrap();
16011        tokio::time::timeout(Duration::from_secs(10), &mut prep)
16012            .await
16013            .unwrap()
16014            .unwrap();
16015        drop(prep);
16016        assert_eq!(runner.state.prep_complete, Some(true));
16017        assert!(root.join("judge-1").exists());
16018        assert!(root.join("advisor-1").exists());
16019        assert_eq!(
16020            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
16021            0
16022        );
16023    }
16024
16025    #[tokio::test]
16026    #[cfg(unix)]
16027    async fn preparation_lock_failure_prevents_repository_changes() {
16028        ask_test_home();
16029        let tmp = tempfile::tempdir().unwrap();
16030        let repo = tmp.path().join("repo");
16031        std::fs::create_dir_all(&repo).unwrap();
16032        init_repo(&repo);
16033        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
16034        // A directory at the required lock path makes opening the real lock
16035        // fail, rather than merely exercising an invalid registration.
16036        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
16037        let mut runner = runner_at(RunStatus::Prep);
16038        runner.state.repo = repo.clone();
16039        runner.state.config.coordination = crate::config::test_coordination();
16040        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16041        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16042        runner.roles.implementers = vec![spec("implement")];
16043        let error = runner.prep().await.unwrap_err();
16044        assert!(
16045            error
16046                .to_string()
16047                .contains("acquire mandatory repository lock")
16048        );
16049        assert!(!runner.state.worktree_root().exists());
16050        assert!(runner.state.candidates.is_empty());
16051        assert!(!repo.join(".git/config.worktree").exists());
16052        assert_eq!(
16053            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
16054            "* main"
16055        );
16056    }
16057
16058    #[tokio::test]
16059    async fn preparation_records_creation_intent_before_git_can_fail() {
16060        ask_test_home();
16061        let tmp = tempfile::tempdir().unwrap();
16062        let repo = tmp.path().join("repo");
16063        std::fs::create_dir_all(&repo).unwrap();
16064        init_repo(&repo);
16065        let mut runner = runner_at(RunStatus::Prep);
16066        runner.state.repo = repo.clone();
16067        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16068        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16069        runner.state.config.blind.commit_msg_hook = false;
16070        runner.roles.implementers = vec![spec("one")];
16071        let label = blind::assign_labels(1, runner.state.seed)[0];
16072        let branch = runner.state.branch_for(label);
16073        // A conflicting branch makes Git refuse creation before a candidate
16074        // can be registered. The durable intent must still identify it.
16075        git::git(&repo, &["branch", &branch]).await.unwrap();
16076        assert!(runner.prep().await.is_err());
16077        let saved = RunState::load(&runner.state.id).unwrap();
16078        assert!(saved.candidates.is_empty());
16079        let event = saved
16080            .events
16081            .iter()
16082            .find(|e| e.node == "prep-intent")
16083            .unwrap();
16084        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
16085        assert_eq!(intent["branch"], branch);
16086        assert_eq!(intent["start"], runner.state.base_commit);
16087        assert_eq!(
16088            intent["worktree"],
16089            runner
16090                .state
16091                .worktree_root()
16092                .join(format!("cand-{label}"))
16093                .to_string_lossy()
16094                .as_ref()
16095        );
16096        assert_eq!(intent["picks"], serde_json::json!([]));
16097    }
16098
16099    #[tokio::test]
16100    async fn invalid_pinned_sources_create_no_partial_worktrees() {
16101        ask_test_home();
16102        let tmp = tempfile::tempdir().unwrap();
16103        let repo = tmp.path().join("repo");
16104        std::fs::create_dir_all(&repo).unwrap();
16105        init_repo(&repo);
16106        let mut runner = runner_at(RunStatus::Prep);
16107        runner.state.repo = repo.clone();
16108        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16109        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16110        runner.state.sources.pick_commits.push(refs::Input {
16111            token: "missing".into(),
16112            sha: "0".repeat(40),
16113            provenance: refs::Provenance::ExplicitCli,
16114        });
16115        assert!(runner.prep().await.is_err());
16116        assert!(runner.state.candidates.is_empty());
16117        assert!(!runner.state.worktree_root().exists());
16118    }
16119
16120    #[tokio::test]
16121    async fn prose_and_legacy_inference_never_import_into_candidates() {
16122        ask_test_home();
16123        let tmp = tempfile::tempdir().unwrap();
16124        let repo = tmp.path().join("repo");
16125        std::fs::create_dir_all(&repo).unwrap();
16126        init_repo(&repo);
16127        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16128        git::git(&repo, &["checkout", "-b", "magi/history/A"])
16129            .await
16130            .unwrap();
16131        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
16132        git::git(&repo, &["add", "."]).await.unwrap();
16133        git::git(&repo, &["commit", "-m", "historical implementation"])
16134            .await
16135            .unwrap();
16136        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
16137        git::git(&repo, &["checkout", "main"]).await.unwrap();
16138        for instruction in [
16139            format!("Example: magi/history/A at {source}"),
16140            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
16141            format!("Never import magi/history/A or {source}"),
16142            format!("Conversation answer: I mentioned {source} only as background"),
16143        ] {
16144            let mut runner = runner_at(RunStatus::Prep);
16145            runner.state.repo = repo.clone();
16146            runner.state.base_commit = base.clone();
16147            runner.state.instruction = instruction.clone();
16148            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
16149            runner.state.config.blind.commit_msg_hook = false;
16150            runner.state.seeds = vec![refs::Seed {
16151                token: "magi/history/A".into(),
16152                sha: source.clone(),
16153                kind: refs::SeedKind::Unmerged,
16154                branch: true,
16155                detail: String::new(),
16156            }];
16157            runner.roles.implementers = vec![spec("one")];
16158            assert_eq!(seeded_instruction(&runner.state), instruction);
16159            // Model an old record with inferred seeds but no structured input field.
16160            let mut legacy = serde_json::to_value(&runner.state).unwrap();
16161            legacy.as_object_mut().unwrap().remove("sources");
16162            runner.state = serde_json::from_value(legacy).unwrap();
16163            assert_eq!(runner.state.sources, refs::Sources::default());
16164            runner.state.save().unwrap();
16165            runner.state = RunState::load(&runner.state.id).unwrap();
16166            runner.prep().await.unwrap();
16167            for candidate in &runner.state.candidates {
16168                assert_eq!(
16169                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
16170                    base
16171                );
16172                assert!(!candidate.worktree.join("unwanted").exists());
16173            }
16174        }
16175    }
16176
16177    #[tokio::test]
16178    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
16179        ask_test_home();
16180        let tmp = tempfile::tempdir().unwrap();
16181        let repo = tmp.path().join("repo");
16182        std::fs::create_dir_all(&repo).unwrap();
16183        init_repo(&repo);
16184        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16185        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
16186        std::fs::write(repo.join("referenced"), "required commit").unwrap();
16187        git::git(&repo, &["add", "."]).await.unwrap();
16188        git::git(&repo, &["commit", "-m", "required seed"])
16189            .await
16190            .unwrap();
16191        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
16192        git::git(&repo, &["checkout", "main"]).await.unwrap();
16193        let mut runner = runner_at(RunStatus::Prep);
16194        runner.state.repo = repo.clone();
16195        runner.state.base_commit = base.clone();
16196        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16197        runner.state.config.graph.advise = false;
16198        runner.state.config.blind.commit_msg_hook = false;
16199        runner.state.sources =
16200            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
16201                .await
16202                .unwrap();
16203        runner.roles.implementers = vec![spec("one"), spec("two")];
16204        runner.roles.judges = vec![spec("judge")];
16205        let label = blind::assign_labels(2, runner.state.seed)[0];
16206        let branch = runner.state.branch_for(label);
16207        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
16208        git::worktree_add_branch(&repo, &worktree, &branch, &base)
16209            .await
16210            .unwrap();
16211        runner.state.candidates.push(Candidate {
16212            index: 0,
16213            label,
16214            agent: "one".into(),
16215            branch,
16216            worktree,
16217            summary: String::new(),
16218            stat: String::new(),
16219            files: 0,
16220            commits: 0,
16221            empty: false,
16222            failed: None,
16223            verified_noop: None,
16224            duration_ms: 0,
16225            folded: false,
16226        });
16227        runner.state.save().unwrap();
16228        // Reload the checkpoint immediately before its first cherry-pick.
16229        runner.state = RunState::load(&runner.state.id).unwrap();
16230        assert_eq!(runner.state.prep_complete, Some(false));
16231        runner.prep().await.unwrap();
16232        assert_eq!(runner.state.prep_complete, Some(true));
16233        assert_eq!(runner.state.candidates.len(), 2);
16234        for candidate in &runner.state.candidates {
16235            assert_eq!(
16236                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16237                git::tree_of(&repo, &seed).await.unwrap()
16238            );
16239            assert_eq!(
16240                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
16241                "required commit"
16242            );
16243        }
16244        assert!(runner.state.worktree_root().join("judge-1").exists());
16245        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
16246        runner.prep().await.unwrap();
16247        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
16248
16249        runner.state.prep_complete = Some(false);
16250        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
16251            run: Some(format!(
16252                "test -f '{}'",
16253                tmp.path().join("dependency-ready").display()
16254            )),
16255            ..Default::default()
16256        }];
16257        assert!(runner.prep().await.is_err());
16258        assert!(runner.state.prep_failure.is_some());
16259        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
16260        runner.state = RunState::load(&runner.state.id).unwrap();
16261        assert!(
16262            runner.prep().await.is_err(),
16263            "correcting the cause alone must not auto-retry"
16264        );
16265        let failed_at = runner.state.events.last().unwrap().at;
16266        assert!(
16267            runner.retry_preparation(failed_at).is_err(),
16268            "old approvals cannot unlock the hold"
16269        );
16270        let mut task = crate::queue::Task::new(
16271            "retry".into(),
16272            "retry setup".into(),
16273            repo.clone(),
16274            crate::queue::Source::Human,
16275        );
16276        task.start(runner.state.id.clone());
16277        task.hold_machine(Some("setup failed".into()));
16278        task.release_operator("cli", None);
16279        runner
16280            .retry_preparation(task.resumption.as_ref().unwrap().at)
16281            .unwrap();
16282        runner.prep().await.unwrap();
16283        assert!(runner.state.prep_failure.is_none());
16284        assert_eq!(runner.state.prep_complete, Some(true));
16285        assert_eq!(runner.state.candidates.len(), 2);
16286        assert!(runner.state.worktree_root().join("judge-1").exists());
16287        for candidate in &runner.state.candidates {
16288            assert_eq!(
16289                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
16290                git::tree_of(&repo, &seed).await.unwrap()
16291            );
16292        }
16293    }
16294
16295    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
16296        let mut heads = Vec::new();
16297        for candidate in candidates {
16298            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
16299        }
16300        heads
16301    }
16302
16303    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
16304        ask_test_home();
16305        let tmp = tempfile::tempdir().unwrap();
16306        let repo = tmp.path().join("repo");
16307        std::fs::create_dir(&repo).unwrap();
16308        init_repo(&repo);
16309        let mut runner = runner_at(RunStatus::Reviewing);
16310        runner.state.repo = repo.clone();
16311        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16312        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16313        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16314            let branch = format!("candidate-{label}");
16315            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
16316            git::worktree_add_branch(&repo, &worktree, &branch, "main")
16317                .await
16318                .unwrap();
16319            if label == 'B' {
16320                std::fs::write(worktree.join("change"), "winner").unwrap();
16321                git::rescue_commit(&worktree, "candidate change")
16322                    .await
16323                    .unwrap();
16324            }
16325            runner.state.candidates.push(Candidate {
16326                index,
16327                label,
16328                agent: "alpha".into(),
16329                branch,
16330                worktree,
16331                summary: "completed reply".into(),
16332                stat: String::new(),
16333                files: usize::from(label == 'B'),
16334                commits: usize::from(label == 'B'),
16335                empty: label == 'A',
16336                failed: None,
16337                verified_noop: None,
16338                duration_ms: 1,
16339                folded: false,
16340            });
16341        }
16342        runner.state.judge_skipped = true;
16343        runner.state.tally = Some(Tally {
16344            first_choice: BTreeMap::new(),
16345            borda: BTreeMap::new(),
16346            winner: 'B',
16347            rankings: 0,
16348            unanimous_initial: false,
16349            deliberated: false,
16350            changed_votes: 0,
16351            unanimous_final: false,
16352            tie_break: None,
16353            judges: 0,
16354            present: 0,
16355            quorum: 0,
16356            met_quorum: true,
16357            uncontested: Some("only B produced a change".into()),
16358        });
16359        runner.state.save().unwrap();
16360        (tmp, runner)
16361    }
16362
16363    #[tokio::test]
16364    async fn fold_losers_reclaims_isolated_outputs_without_a_pool_limit() {
16365        let (_tmp, mut runner) = preservation_resume_fixture().await;
16366        runner.state.config.verify.build_cache = crate::config::BuildCache::Isolated;
16367        runner.state.config.disk.cache_limit_bytes = 0;
16368        runner.state.config.graph.advise = true;
16369        runner.state.config.graph.advisors = 1;
16370        let advisor = runner.state.worktree_root().join("advisor-1");
16371        git::worktree_add_detached(&runner.state.repo, &advisor, "main")
16372            .await
16373            .unwrap();
16374        let home = crate::run::home();
16375        let loser =
16376            crate::build_target::resolve(&home, &runner.state.candidates[0].worktree).unwrap();
16377        let winner =
16378            crate::build_target::resolve(&home, &runner.state.candidates[1].worktree).unwrap();
16379        let advisor = crate::build_target::resolve(&home, &advisor).unwrap();
16380        for target in [&loser, &winner, &advisor] {
16381            std::fs::write(target.join("build-output"), "compiled").unwrap();
16382        }
16383
16384        runner.fold_losers().await.unwrap();
16385
16386        assert!(!loser.join("build-output").exists());
16387        assert!(!advisor.join("build-output").exists());
16388        assert!(winner.join("build-output").is_file());
16389    }
16390
16391    #[tokio::test]
16392    async fn fold_losers_persists_discard_before_later_teardown_failure() {
16393        let (_tmp, mut runner) = preservation_resume_fixture().await;
16394        runner.state.candidates[0].folded = true;
16395        runner.state.config.graph.advise = true;
16396        runner.state.config.graph.advisors = 2;
16397        let repo = runner.state.repo.clone();
16398        let root = runner.state.worktree_root();
16399        let discarded = root.join("advisor-1");
16400        git::worktree_add_detached(&repo, &discarded, "main")
16401            .await
16402            .unwrap();
16403        std::fs::write(discarded.join("design_proposal.json"), "{}").unwrap();
16404        let protected = root.join("advisor-2");
16405        git::worktree_add_branch(&repo, &protected, "attached-advisor", "main")
16406            .await
16407            .unwrap();
16408        std::fs::write(protected.join("keep"), "uncommitted").unwrap();
16409        runner.state.save().unwrap();
16410
16411        assert!(runner.fold_losers().await.is_err());
16412        assert!(!discarded.exists());
16413        assert!(protected.join("keep").exists());
16414        let saved = RunState::load(&runner.state.id).unwrap();
16415        assert!(saved.events.iter().any(|event| {
16416            event.message.contains("advisor-1") && event.message.contains("design_proposal.json")
16417        }));
16418    }
16419
16420    #[tokio::test]
16421    async fn read_seat_teardown_discards_only_owned_detached_seats() {
16422        let (tmp, mut runner) = preservation_resume_fixture().await;
16423        let repo = runner.state.repo.clone();
16424        let root = runner.state.worktree_root();
16425        for name in ["advisor-3", "judge-1", "review-2"] {
16426            let path = root.join(name);
16427            git::worktree_add_detached(&repo, &path, "main")
16428                .await
16429                .unwrap();
16430            std::fs::write(path.join("README.md"), "uncommitted edit").unwrap();
16431            std::fs::write(path.join("design_proposal.json"), "{}").unwrap();
16432            remove_read_seat_worktree(&mut runner.state, &path)
16433                .await
16434                .unwrap();
16435            assert!(!path.exists());
16436        }
16437        let outside = tmp.path().join("advisor-1");
16438        let unknown = root.join("unknown");
16439        let attached = root.join("advisor-4");
16440        git::worktree_add_detached(&repo, &outside, "main")
16441            .await
16442            .unwrap();
16443        git::worktree_add_detached(&repo, &unknown, "main")
16444            .await
16445            .unwrap();
16446        git::worktree_add_branch(&repo, &attached, "attached-advisor", "main")
16447            .await
16448            .unwrap();
16449        let foreign = tmp.path().join("foreign");
16450        std::fs::create_dir(&foreign).unwrap();
16451        init_repo(&foreign);
16452        let wrong_repo = root.join("judge-2");
16453        git::worktree_add_detached(&foreign, &wrong_repo, "main")
16454            .await
16455            .unwrap();
16456        let candidate = runner.state.candidates[0].worktree.clone();
16457        for path in [outside, unknown, attached, wrong_repo, candidate] {
16458            std::fs::write(path.join("uncommitted"), "keep").unwrap();
16459            assert!(
16460                remove_read_seat_worktree(&mut runner.state, &path)
16461                    .await
16462                    .is_err()
16463            );
16464            assert_eq!(
16465                std::fs::read_to_string(path.join("uncommitted")).unwrap(),
16466                "keep"
16467            );
16468        }
16469    }
16470
16471    #[tokio::test]
16472    #[cfg(unix)]
16473    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
16474        for (solo, failed_checkpoint) in
16475            [(true, false), (false, false), (true, true), (false, true)]
16476        {
16477            let (_tmp, mut runner) = preservation_resume_fixture().await;
16478            if solo {
16479                runner.state.candidates.retain(|c| c.label == 'A');
16480            }
16481            runner.state.candidates[0].empty = false;
16482            runner.state.candidates[0].summary.clear();
16483            runner.state.tally = None;
16484            runner.state.judge_skipped = false;
16485            runner.state.status = RunStatus::Implementing;
16486            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16487            let mut implementer = spec("alpha");
16488            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16489            runner.roles.implementers = vec![implementer];
16490            let patch = runner.state.dir().join("artifacts/cand-A.patch");
16491            if failed_checkpoint {
16492                std::fs::create_dir_all(&patch).unwrap();
16493            }
16494            runner.state.save().unwrap();
16495            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
16496            drop(pressure);
16497            runner.state = RunState::load(&runner.state.id).unwrap();
16498            assert_eq!(runner.state.status, RunStatus::Blocked);
16499            assert!(runner.state.interrupted_implementations.contains(&'A'));
16500            assert_eq!(runner.state.candidates[0].commits, 0);
16501            assert!(!runner.state.candidates[0].empty);
16502            assert!(runner.state.candidates[0].failed.is_none());
16503            if failed_checkpoint {
16504                assert!(runner.state.candidate_preservation.contains_key(&'A'));
16505                std::fs::remove_dir(patch).unwrap();
16506            }
16507            runner.state.resource_block = None;
16508            runner.state.config.disk.min_free_bytes = 0;
16509            runner.roles.implementers[0].command = vec![
16510                "sh".into(),
16511                "-c".into(),
16512                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
16513            ];
16514            runner.implement().await.unwrap();
16515            assert!(runner.state.interrupted_implementations.is_empty());
16516            assert!(runner.state.candidate_preservation.is_empty());
16517            assert_eq!(runner.state.candidates[0].commits, 1);
16518            assert!(!runner.state.candidates[0].empty);
16519            assert!(
16520                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
16521                    .unwrap()
16522                    .contains("retried empty interrupted seat")
16523            );
16524            if !solo {
16525                assert_eq!(runner.state.candidates[1].commits, 1);
16526            }
16527        }
16528    }
16529
16530    #[tokio::test]
16531    #[cfg(unix)]
16532    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
16533        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
16534            let (_tmp, mut runner) = preservation_resume_fixture().await;
16535            runner.state.candidates.retain(|c| c.label == 'A');
16536            runner.state.candidates[0].empty = false;
16537            runner.state.candidates[0].summary.clear();
16538            runner.state.tally = None;
16539            runner.state.judge_skipped = false;
16540            runner.state.status = RunStatus::Implementing;
16541            // Force only the real post-spawn watchdog, without failing the
16542            // separate admission check. Existing edits predate its measurement.
16543            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
16544            let wt = runner.state.candidates[0].worktree.clone();
16545            std::fs::write(
16546                wt.join("interrupted-source"),
16547                "retain implementation work\n",
16548            )
16549            .unwrap();
16550            if committed {
16551                git::rescue_commit(&wt, "implementation checkpoint")
16552                    .await
16553                    .unwrap();
16554            }
16555            let lock = git::git(
16556                &wt,
16557                &[
16558                    "rev-parse",
16559                    "--path-format=absolute",
16560                    "--git-path",
16561                    "index.lock",
16562                ],
16563            )
16564            .await
16565            .unwrap();
16566            if block_rescue {
16567                std::fs::write(&lock, "fixture-owned lock").unwrap();
16568            }
16569            let mut implementer = spec("alpha");
16570            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
16571            runner.roles.implementers = vec![implementer];
16572            runner.state.save().unwrap();
16573            let result = runner.implement().await;
16574            drop(pressure);
16575            assert_eq!(result.is_err(), block_rescue);
16576            assert_eq!(
16577                runner.state.resource_block.as_ref().unwrap().kind,
16578                "disk pressure"
16579            );
16580            assert!(
16581                runner
16582                    .state
16583                    .events
16584                    .iter()
16585                    .any(|e| e.message.contains("agent execution interrupted"))
16586            );
16587            assert!(
16588                runner
16589                    .state
16590                    .dir()
16591                    .join("artifacts/candidate-A-recovery/staged.patch")
16592                    .exists()
16593            );
16594            if block_rescue {
16595                let saved = RunState::load(&runner.state.id).unwrap();
16596                assert!(saved.candidate_preservation.contains_key(&'A'));
16597                assert!(!saved.candidates[0].empty);
16598                assert_eq!(
16599                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
16600                    "retain implementation work\n"
16601                );
16602                std::fs::remove_file(lock).unwrap();
16603                runner.state = saved;
16604            } else {
16605                assert_eq!(runner.state.status, RunStatus::Blocked);
16606                assert_eq!(runner.state.candidates[0].commits, 1);
16607            }
16608            // Recovery must save the candidate before deciding whether another
16609            // implementation is needed. No implementer is available on retry.
16610            runner.roles.implementers.clear();
16611            runner.state.config.disk.min_free_bytes = 0;
16612            runner.state.resource_block = None;
16613            runner.implement().await.unwrap();
16614            assert!(runner.state.candidate_preservation.is_empty());
16615            assert_eq!(runner.state.candidates[0].commits, 1);
16616            assert!(!runner.state.candidates[0].empty);
16617            assert!(git::is_clean(&wt).await.unwrap());
16618            assert!(
16619                crate::run::read_artifact(&runner.state, "cand-A.patch")
16620                    .unwrap()
16621                    .contains("retain implementation work")
16622            );
16623        }
16624    }
16625
16626    #[tokio::test]
16627    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
16628        let (_tmp, mut runner) = preservation_resume_fixture().await;
16629        // Schema 15 already confirmed the unfolded no-change result.
16630        runner.implement().await.unwrap();
16631        assert!(runner.state.candidate_preservation.is_empty());
16632        assert!(
16633            !runner
16634                .state
16635                .dir()
16636                .join("artifacts/candidate-A-recovery")
16637                .exists()
16638        );
16639        runner.fold_losers().await.unwrap();
16640        assert!(!runner.state.candidates[0].worktree.exists());
16641        for schema in [14, crate::run::SCHEMA] {
16642            runner.state.schema = schema;
16643            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16644            runner.state.save().unwrap();
16645            runner.state = RunState::load(&runner.state.id).unwrap();
16646            runner.implement().await.unwrap();
16647            assert!(runner.state.candidate_preservation.is_empty());
16648            assert!(runner.state.candidates[0].folded);
16649            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16650        }
16651    }
16652
16653    #[tokio::test]
16654    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
16655        let (_tmp, mut runner) = preservation_resume_fixture().await;
16656        runner.state.candidates.retain(|c| c.label == 'A');
16657        runner.state.tally = None;
16658        runner.state.judge_skipped = false;
16659        let candidate = &mut runner.state.candidates[0];
16660        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
16661        candidate.failed = Some("deployment validation unavailable".into());
16662        std::fs::write(
16663            candidate.worktree.join("repair"),
16664            "future launcher policy\n",
16665        )
16666        .unwrap();
16667        runner
16668            .state
16669            .candidate_preservation
16670            .insert('A', "awaiting persistence".into());
16671        runner.persist_candidates().await.unwrap();
16672        let candidate = &runner.state.candidates[0];
16673        assert!(!candidate.empty);
16674        assert_eq!(candidate.commits, 1);
16675        assert_eq!(candidate.files, 1);
16676        assert!(candidate.failed.is_none());
16677        assert!(candidate.summary.contains("EPERM"));
16678        assert!(
16679            crate::run::read_artifact(&runner.state, "cand-A.patch")
16680                .unwrap()
16681                .contains("future launcher policy")
16682        );
16683        assert!(runner.state.candidate_preservation.is_empty());
16684    }
16685
16686    #[tokio::test]
16687    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16688        let (_tmp, mut runner) = preservation_resume_fixture().await;
16689        let wt = runner.state.candidates[0].worktree.clone();
16690        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16691        runner.state.schema = 14;
16692        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16693        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16694        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16695        for _ in 0..2 {
16696            assert!(runner.implement().await.is_err());
16697            assert_eq!(runner.state.status, RunStatus::Blocked);
16698            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16699            assert!(
16700                runner.state.candidates[0].empty,
16701                "old verdict must remain unusable"
16702            );
16703            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16704            assert!(runner.state.judge_skipped);
16705            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16706            assert!(
16707                crate::run::read_artifact(&runner.state, "cand-A.patch")
16708                    .unwrap()
16709                    .contains("owned edits")
16710            );
16711            assert!(wt.exists());
16712            assert_eq!(
16713                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16714                    .await
16715                    .unwrap(),
16716                1
16717            );
16718            runner.state = RunState::load(&runner.state.id).unwrap();
16719        }
16720    }
16721
16722    #[tokio::test]
16723    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16724        let (_tmp, mut runner) = preservation_resume_fixture().await;
16725        let wt = runner.state.candidates[0].worktree.clone();
16726        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16727        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16728        let index_lock = PathBuf::from(
16729            git::git(
16730                &wt,
16731                &[
16732                    "rev-parse",
16733                    "--path-format=absolute",
16734                    "--git-path",
16735                    "index.lock",
16736                ],
16737            )
16738            .await
16739            .unwrap(),
16740        );
16741        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16742        runner.state.schema = 14;
16743        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16744        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16745        assert!(runner.implement().await.is_err());
16746        assert!(runner.after_implement().is_err());
16747        assert!(runner.fold_losers().await.is_err());
16748        assert_eq!(
16749            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16750            "failed seat edits"
16751        );
16752        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16753        runner.state = RunState::load(&runner.state.id).unwrap();
16754        std::fs::remove_file(index_lock).unwrap();
16755        // Recovery finds changes that the old uncontested result never judged.
16756        assert!(runner.implement().await.is_err());
16757        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16758        assert!(runner.state.candidates[0].empty);
16759        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16760        assert!(
16761            crate::run::read_artifact(&runner.state, "cand-A.patch")
16762                .unwrap()
16763                .contains("failed seat edits")
16764        );
16765        assert!(wt.exists());
16766    }
16767
16768    #[tokio::test]
16769    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16770        let (_tmp, mut runner) = preservation_resume_fixture().await;
16771        let repo = runner.state.repo.clone();
16772        let common = git::git(
16773            &repo,
16774            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16775        )
16776        .await
16777        .unwrap();
16778        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16779        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16780        assert!(runner.fold_losers().await.is_err());
16781        assert!(!runner.state.candidates[0].worktree.exists());
16782        assert!(!runner.state.candidates[0].folded);
16783        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16784        runner.state = RunState::load(&runner.state.id).unwrap();
16785        std::fs::remove_file(branch_lock).unwrap();
16786        let judge = runner.state.worktree_root().join("judge-1");
16787        git::worktree_add_detached(&repo, &judge, "main")
16788            .await
16789            .unwrap();
16790        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16791        git::git(&repo, &["worktree", "lock", &git::path_arg(&judge)])
16792            .await
16793            .unwrap();
16794        runner.roles.judges = vec![spec("judge")];
16795        assert!(runner.fold_losers().await.is_err());
16796        runner.state = RunState::load(&runner.state.id).unwrap();
16797        assert!(
16798            runner.state.candidates[0].folded,
16799            "candidate progress saved before judge failure"
16800        );
16801        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16802        assert!(judge.join("uncommitted").exists());
16803        git::git(&repo, &["worktree", "unlock", &git::path_arg(&judge)])
16804            .await
16805            .unwrap();
16806        runner.fold_losers().await.unwrap();
16807        assert!(!judge.exists());
16808        runner.fold_losers().await.unwrap();
16809    }
16810
16811    #[tokio::test]
16812    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16813        let (_tmp, mut runner) = preservation_resume_fixture().await;
16814        runner.state.candidates.retain(|c| c.label == 'B');
16815        runner.state.tally = None;
16816        runner.state.judge_skipped = false;
16817        runner.state.prep_complete = Some(true);
16818        runner.state.config.graph.advise = false;
16819        runner.state.config.graph.selected_implementer = Some("alpha".into());
16820        let wt = runner.state.candidates[0].worktree.clone();
16821        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16822        runner
16823            .state
16824            .candidate_preservation
16825            .insert('B', "awaiting persistence".into());
16826        let index_lock = git::git(
16827            &wt,
16828            &[
16829                "rev-parse",
16830                "--path-format=absolute",
16831                "--git-path",
16832                "index.lock",
16833            ],
16834        )
16835        .await
16836        .unwrap();
16837        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16838        runner.state.save().unwrap();
16839        assert!(runner.execute_graph().await.is_err());
16840        assert_eq!(runner.state.status, RunStatus::Blocked);
16841        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16842        std::fs::remove_file(index_lock).unwrap();
16843        // Park at the next graph boundary: this tests the real selected-seat
16844        // hold guard without dispatching reviewers or publishing anything.
16845        for _ in 0..2 {
16846            runner.state = RunState::load(&runner.state.id).unwrap();
16847            runner.pause.park();
16848            runner.execute_graph().await.unwrap();
16849            assert_eq!(runner.state.status, RunStatus::Implementing);
16850            assert!(runner.state.candidate_preservation.is_empty());
16851            assert_eq!(runner.state.candidates[0].commits, 2);
16852            assert!(runner.state.tally.is_none());
16853        }
16854        runner.implement().await.unwrap();
16855        assert_eq!(runner.state.status, RunStatus::Judging);
16856        // A genuine selected-implementer hold must still require release.
16857        runner.state.status = RunStatus::Blocked;
16858        runner.state.candidates[0].failed = Some("implementation failed".into());
16859        runner.state.save().unwrap();
16860        let error = runner.execute_graph().await.unwrap_err();
16861        assert!(
16862            error
16863                .to_string()
16864                .contains("selected implementation is held")
16865        );
16866    }
16867
16868    #[tokio::test]
16869    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16870        let (_tmp, mut runner) = preservation_resume_fixture().await;
16871        let repo = runner.state.repo.clone();
16872        let candidate = runner.state.candidates[0].clone();
16873        // Remove only the disposable fixture directory, leaving Git's linked
16874        // worktree registration and branch intact, as in a vanished checkout.
16875        let registered = git::worktree_holding(&repo, &candidate.branch)
16876            .await
16877            .unwrap()
16878            .unwrap();
16879        assert_eq!(
16880            registered.canonicalize().unwrap(),
16881            candidate.worktree.canonicalize().unwrap()
16882        );
16883        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16884        assert_eq!(
16885            git::worktree_holding(&repo, &candidate.branch)
16886                .await
16887                .unwrap(),
16888            Some(registered)
16889        );
16890        assert!(
16891            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16892            "the stale registration initially keeps the branch checked out"
16893        );
16894        runner.fold_losers().await.unwrap();
16895        assert!(runner.state.candidates[0].folded);
16896        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16897        assert_eq!(
16898            git::worktree_holding(&repo, &candidate.branch)
16899                .await
16900                .unwrap(),
16901            None
16902        );
16903        runner.state = RunState::load(&runner.state.id).unwrap();
16904        runner.fold_losers().await.unwrap();
16905    }
16906
16907    #[tokio::test]
16908    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16909        let (_tmp, mut runner) = preservation_resume_fixture().await;
16910        let repo = runner.state.repo.clone();
16911        let candidate = runner.state.candidates[0].clone();
16912        let winner = runner.state.candidates[1].clone();
16913        let registered = git::worktree_holding(&repo, &candidate.branch)
16914            .await
16915            .unwrap()
16916            .unwrap();
16917        assert_eq!(
16918            registered.canonicalize().unwrap(),
16919            candidate.worktree.canonicalize().unwrap()
16920        );
16921        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16922        assert_eq!(
16923            git::worktree_holding(&repo, &candidate.branch)
16924                .await
16925                .unwrap(),
16926            Some(registered)
16927        );
16928        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16929
16930        for _ in 0..2 {
16931            fold_run(&mut runner.state, false, &crate::run::home())
16932                .await
16933                .unwrap();
16934            runner.state = RunState::load(&runner.state.id).unwrap();
16935            assert!(runner.state.candidates[0].folded);
16936            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16937            assert_eq!(
16938                git::worktree_holding(&repo, &candidate.branch)
16939                    .await
16940                    .unwrap(),
16941                None
16942            );
16943            assert!(!runner.state.candidates[1].folded);
16944            assert!(winner.worktree.exists());
16945            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16946        }
16947    }
16948    #[tokio::test]
16949    async fn fold_recovery_override_archives_sources_and_keeps_safety_checks() {
16950        for sidecar in [false, true] {
16951            let tmp = tempfile::tempdir().unwrap();
16952            let home = tmp.path().join("home");
16953            let repo = tmp.path().join("repo");
16954            std::fs::create_dir(&repo).unwrap();
16955            init_repo(&repo);
16956            let mut state = runner_at(RunStatus::Superseded).state;
16957            state.repo = repo.clone();
16958            state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16959            state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16960            state.driver_exited = true;
16961            state.reserve_under(&home).unwrap();
16962            git::git(
16963                &repo,
16964                &["update-ref", "refs/remotes/origin/main", &state.base_commit],
16965            )
16966            .await
16967            .unwrap();
16968            let wt = state.worktree_root().join("cand-A");
16969            git::worktree_add_branch(&repo, &wt, "candidate", "main")
16970                .await
16971                .unwrap();
16972            std::fs::write(wt.join("independent"), "retained implementation\n").unwrap();
16973            git::commit_all(&wt, "fixture: independent source")
16974                .await
16975                .unwrap();
16976            let head = git::rev_parse(&wt, "HEAD").await.unwrap();
16977            state.candidates.push(Candidate {
16978                index: 0,
16979                label: 'A',
16980                agent: "fixture".into(),
16981                branch: "candidate".into(),
16982                worktree: wt.clone(),
16983                summary: String::new(),
16984                stat: String::new(),
16985                files: 0,
16986                commits: 0,
16987                empty: false,
16988                failed: None,
16989                verified_noop: None,
16990                duration_ms: 0,
16991                folded: false,
16992            });
16993            let mut task = crate::queue::Task::new(
16994                "task".into(),
16995                "task".into(),
16996                repo.clone(),
16997                crate::queue::Source::Human,
16998            );
16999            task.runs.push(state.id.clone());
17000            crate::queue::Queue::at(home.join("queue"))
17001                .put(&mut task)
17002                .unwrap();
17003            if sidecar {
17004                crate::pr_recovery::retain_sources(&task, &home).unwrap();
17005            }
17006            state.save_under(&home).unwrap();
17007            let reason = crate::pr_recovery::source_retention_reason(&home, &state).unwrap();
17008            assert!(
17009                fold_run(&mut state, true, &home)
17010                    .await
17011                    .unwrap_err()
17012                    .to_string()
17013                    .contains(&reason)
17014            );
17015            let mut disk = state.config.disk.clone();
17016            disk.fold_grace_secs = 0;
17017            assert_eq!(
17018                crate::clean::fold_due(
17019                    &home.join("runs"),
17020                    &home,
17021                    &state.worktree_root(),
17022                    &disk,
17023                    Timestamp::now() + std::time::Duration::from_secs(10),
17024                )
17025                .await
17026                .unwrap(),
17027                (0, 0)
17028            );
17029            assert!(wt.exists());
17030            state.status = RunStatus::Blocked;
17031            state.save_under(&home).unwrap();
17032            assert!(
17033                fold_run_overriding_recovery_retention(&mut state, true, &home)
17034                    .await
17035                    .is_err()
17036            );
17037            state.status = RunStatus::Superseded;
17038            state.driver_exited = false;
17039            state.driver_claimed = None;
17040            state.save_under(&home).unwrap();
17041            assert!(
17042                fold_run_overriding_recovery_retention(&mut state, true, &home)
17043                    .await
17044                    .is_err()
17045            );
17046            state.driver_pid = Some(std::process::id());
17047            state.driver_started_at = crate::proc::process_started_at(std::process::id());
17048            state.save_under(&home).unwrap();
17049            assert!(
17050                fold_run_overriding_recovery_retention(&mut state, true, &home)
17051                    .await
17052                    .is_err()
17053            );
17054            state.driver_exited = true;
17055            state.driver_pid = None;
17056            state.driver_started_at = None;
17057            state.save_under(&home).unwrap();
17058            let lease = crate::driver::maintenance(&home, &state.id).unwrap();
17059            assert!(
17060                fold_run_overriding_recovery_retention(&mut state, true, &home)
17061                    .await
17062                    .is_err()
17063            );
17064            drop(lease);
17065            #[cfg(unix)]
17066            {
17067                let output = crate::retention::OutputLease::take(&wt, false).unwrap();
17068                assert!(
17069                    fold_run_overriding_recovery_retention(&mut state, true, &home)
17070                        .await
17071                        .is_err()
17072                );
17073                drop(output);
17074            }
17075            let owner = crate::pr_head::owner_lock(&home, &state.id).unwrap();
17076            assert!(
17077                fold_run_overriding_recovery_retention(&mut state, true, &home)
17078                    .await
17079                    .is_err()
17080            );
17081            drop(owner);
17082            let checkpoint = git::git(
17083                &wt,
17084                &[
17085                    "rev-parse",
17086                    "--path-format=absolute",
17087                    "--git-path",
17088                    "CHERRY_PICK_HEAD",
17089                ],
17090            )
17091            .await
17092            .unwrap();
17093            std::fs::write(&checkpoint, &head).unwrap();
17094            assert!(
17095                fold_run_overriding_recovery_retention(&mut state, true, &home)
17096                    .await
17097                    .is_err()
17098            );
17099            std::fs::remove_file(checkpoint).unwrap();
17100            std::fs::write(wt.join("dirty"), "keep").unwrap();
17101            assert!(
17102                fold_run_overriding_recovery_retention(&mut state, true, &home)
17103                    .await
17104                    .is_err()
17105            );
17106            assert!(wt.join("dirty").exists());
17107            std::fs::remove_file(wt.join("dirty")).unwrap();
17108            let evidence = home
17109                .join("runs")
17110                .join(&state.id)
17111                .join("artifacts/fold-evidence");
17112            std::fs::create_dir_all(evidence.parent().unwrap()).unwrap();
17113            std::fs::write(&evidence, "blocked archive").unwrap();
17114            assert!(
17115                fold_run_overriding_recovery_retention(&mut state, true, &home)
17116                    .await
17117                    .is_err()
17118            );
17119            assert!(wt.exists());
17120            assert!(
17121                !RunState::load_under(&state.id, &home)
17122                    .unwrap()
17123                    .events
17124                    .iter()
17125                    .any(|event| event.message.contains("--override-recovery-retention"))
17126            );
17127            std::fs::remove_file(&evidence).unwrap();
17128            fold_run_overriding_recovery_retention(&mut state, true, &home)
17129                .await
17130                .unwrap();
17131            assert!(!wt.exists());
17132            assert!(!git::branch_exists(&repo, "candidate").await.unwrap());
17133            assert!(
17134                git::worktree_holding(&repo, "candidate")
17135                    .await
17136                    .unwrap()
17137                    .is_none()
17138            );
17139            assert!(
17140                std::fs::read_to_string(evidence.join(format!("{head}.patch")))
17141                    .unwrap()
17142                    .contains("retained implementation")
17143            );
17144            assert!(evidence.join(format!("{head}.index")).is_file());
17145            let saved = RunState::load_under(&state.id, &home).unwrap();
17146            assert_eq!(saved.status, RunStatus::Superseded);
17147            assert!(saved.candidates[0].folded);
17148            assert!(saved.events.iter().any(|event| {
17149                event.message.contains("--override-recovery-retention")
17150                    && event.message.contains(&state.id)
17151                    && event.message.contains(&reason)
17152            }));
17153            assert_eq!(
17154                crate::pr_recovery::explicit_sources_retained(&home, &state.id),
17155                sidecar
17156            );
17157        }
17158    }
17159
17160    #[tokio::test]
17161    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
17162        ask_test_home();
17163        let tmp = tempfile::tempdir().unwrap();
17164        let repo = tmp.path().join("repo");
17165        std::fs::create_dir(&repo).unwrap();
17166        init_repo(&repo);
17167        let mut runner = runner_at(RunStatus::Implementing);
17168        runner.state.repo = repo.clone();
17169        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
17170        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
17171        for (index, label) in ['A', 'B'].into_iter().enumerate() {
17172            let branch = format!("candidate-{label}");
17173            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
17174            git::worktree_add_branch(&repo, &wt, &branch, "main")
17175                .await
17176                .unwrap();
17177            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
17178            git::git(&wt, &["add", "README.md"]).await.unwrap();
17179            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
17180            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
17181            runner.state.candidates.push(Candidate {
17182                index,
17183                label,
17184                agent: "alpha".to_owned(),
17185                branch,
17186                worktree: wt,
17187                summary: "completed implementation".to_owned(),
17188                stat: String::new(),
17189                files: 0,
17190                commits: 0,
17191                empty: false,
17192                failed: None,
17193                verified_noop: None,
17194                duration_ms: 42,
17195                folded: false,
17196            });
17197            runner
17198                .state
17199                .candidate_preservation
17200                .insert(label, "awaiting persistence".to_owned());
17201        }
17202        let wt = runner.state.candidates[0].worktree.clone();
17203        let index_lock = git::git(
17204            &wt,
17205            &[
17206                "rev-parse",
17207                "--path-format=absolute",
17208                "--git-path",
17209                "index.lock",
17210            ],
17211        )
17212        .await
17213        .unwrap();
17214        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
17215        runner.state.save().unwrap();
17216        assert!(runner.persist_candidates().await.is_err());
17217        assert!(runner.state.candidate_preservation.contains_key(&'A'));
17218        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
17219        assert_eq!(
17220            runner.state.candidates[1].commits, 1,
17221            "other results are still collected"
17222        );
17223        assert!(
17224            !runner.state.candidates[0].empty,
17225            "failed rescue is not no change"
17226        );
17227        assert_eq!(
17228            runner.state.candidates[0].summary,
17229            "completed implementation"
17230        );
17231        assert!(runner.after_implement().is_err());
17232        let home = tmp.path().join("fold-home");
17233        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
17234        assert!(wt.join("new-file").exists());
17235        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
17236        assert!(recovery.join("staged.patch").exists());
17237        assert!(recovery.join("unstaged.patch").exists());
17238        assert_eq!(
17239            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
17240            "untracked edits\n"
17241        );
17242        runner.state.begin_pass();
17243        runner.state.end_pass_parked(ParkCause::Upgrade {
17244            from: "1.0".into(),
17245            to: Some("1.1".into()),
17246        });
17247        runner.state.save().unwrap();
17248        runner.state = RunState::load(&runner.state.id).unwrap();
17249        assert!(matches!(
17250            runner.state.passes.last().unwrap().end,
17251            Some(crate::run::PassEnd::Parked {
17252                cause: ParkCause::Upgrade { .. }
17253            })
17254        ));
17255        std::fs::remove_file(index_lock).unwrap();
17256        // No implementer is configured: a persistence retry cannot dispatch a seat.
17257        runner.implement().await.unwrap();
17258        assert!(runner.state.candidate_preservation.is_empty());
17259        assert!(
17260            runner
17261                .state
17262                .candidates
17263                .iter()
17264                .all(|c| c.commits == 1 && !c.empty)
17265        );
17266        assert!(git::is_clean(&wt).await.unwrap());
17267        assert!(
17268            crate::run::read_artifact(&runner.state, "cand-A.patch")
17269                .unwrap()
17270                .contains("later tracked edits")
17271        );
17272    }
17273
17274    #[tokio::test]
17275    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
17276        ask_test_home();
17277        let tmp = tempfile::tempdir().unwrap();
17278        let repo = tmp.path().join("repo");
17279        std::fs::create_dir(&repo).unwrap();
17280        init_repo(&repo);
17281        let wt = tmp.path().join("candidate");
17282        git::worktree_add_branch(&repo, &wt, "candidate", "main")
17283            .await
17284            .unwrap();
17285        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17286        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
17287        assert!(wt.join("uncommitted").exists());
17288        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17289        std::fs::remove_file(wt.join("uncommitted")).unwrap();
17290        let mut runner = runner_at(RunStatus::Implementing);
17291        runner.state.repo = repo.clone();
17292        runner.state.base_commit = "missing-base".to_owned();
17293        runner.state.candidates.push(Candidate {
17294            index: 0,
17295            label: 'A',
17296            agent: "alpha".into(),
17297            branch: "candidate".into(),
17298            worktree: wt.clone(),
17299            summary: "completed".into(),
17300            stat: String::new(),
17301            files: 0,
17302            commits: 0,
17303            empty: false,
17304            failed: None,
17305            verified_noop: None,
17306            duration_ms: 1,
17307            folded: false,
17308        });
17309        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
17310        assert!(
17311            fold_run(&mut runner.state, true, &tmp.path().join("home"))
17312                .await
17313                .is_err()
17314        );
17315        assert!(!runner.state.candidates[0].folded);
17316        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
17317        std::fs::remove_file(wt.join("uncommitted")).unwrap();
17318        runner
17319            .state
17320            .candidate_preservation
17321            .insert('A', "pending".into());
17322        assert!(runner.persist_candidates().await.is_err());
17323        assert!(!runner.state.candidates[0].empty);
17324        assert!(!runner.state.candidates[0].folded);
17325        assert!(wt.exists());
17326        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
17327        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
17328        std::fs::create_dir_all(&patch_path).unwrap();
17329        assert!(
17330            runner.persist_candidates().await.is_err(),
17331            "artifact failure stays pending"
17332        );
17333        assert!(runner.state.candidate_preservation.contains_key(&'A'));
17334        assert!(!runner.state.candidates[0].empty);
17335        std::fs::remove_dir(patch_path).unwrap();
17336        runner.persist_candidates().await.unwrap();
17337        assert!(
17338            runner.state.candidates[0].empty,
17339            "only successful inspection confirms no change"
17340        );
17341    }
17342
17343    fn init_repo(dir: &Path) {
17344        let run = |args: &[&str]| {
17345            let out = std::process::Command::new("git")
17346                .args(args)
17347                .current_dir(dir)
17348                .quiet()
17349                .output()
17350                .expect("spawn git");
17351            assert!(
17352                out.status.success(),
17353                "git {args:?} failed: {}",
17354                String::from_utf8_lossy(&out.stderr)
17355            );
17356        };
17357        run(&["init", "-b", "main"]);
17358        run(&["config", "user.name", "magi test"]);
17359        run(&["config", "user.email", "magi@example.com"]);
17360        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
17361        run(&["add", "-A"]);
17362        run(&["commit", "-m", "init"]);
17363    }
17364
17365    // `settle_questions` is what closes the ghost the phone showed: a run's
17366    // seat asked something, the run then ended, and nothing was left to
17367    // abandon the question it left `open`. `HOME` is a process-wide
17368    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
17369    // first time it runs in the binary — every test below still reaches the
17370    // same directory whichever call won, and each gets its own run id from
17371    // `RunState::new`, so they never collide there.
17372    fn ask_test_home() {
17373        crate::run::pin_test_home();
17374    }
17375
17376    #[test]
17377    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
17378        ask_test_home();
17379        let mut runner = runner_at(RunStatus::Prep);
17380        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17381        // A blocked temporary output path makes the terminal save fail while
17382        // preserving the already published reservation.
17383        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
17384        let error = guard
17385            .finish(
17386                &mut runner.state,
17387                Err(
17388                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
17389                        .into(),
17390                ),
17391            )
17392            .unwrap_err();
17393        assert_eq!(
17394            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17395            std::io::ErrorKind::PermissionDenied
17396        );
17397        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
17398        let message = format!("{error:#}");
17399        assert!(message.contains("original refusal"));
17400        assert!(!message.contains("could not save terminal"));
17401        assert_eq!(
17402            error.chain().count(),
17403            2,
17404            "only startup attribution wraps the original error"
17405        );
17406        let saved = RunState::load(&runner.state.id).unwrap();
17407        assert_eq!(saved.status, RunStatus::Blocked);
17408        assert!(!saved.startup.unwrap().complete);
17409    }
17410
17411    #[test]
17412    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
17413        ask_test_home();
17414        let mut runner = runner_at(RunStatus::Prep);
17415        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17416        let id = runner.state.id.clone();
17417        drop(guard);
17418        let saved = RunState::load(&id).unwrap();
17419        assert_eq!(saved.status, RunStatus::Blocked);
17420        assert!(!saved.startup.as_ref().unwrap().complete);
17421        assert!(
17422            saved
17423                .startup
17424                .unwrap()
17425                .failure
17426                .unwrap()
17427                .contains("interrupted")
17428        );
17429        assert!(Runner::resume(&id).is_err());
17430    }
17431
17432    #[test]
17433    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
17434        ask_test_home();
17435        let mut runner = runner_at(RunStatus::Prep);
17436        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
17437        let error = guard
17438            .finish(
17439                &mut runner.state,
17440                Err(
17441                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
17442                        .into(),
17443                ),
17444            )
17445            .unwrap_err();
17446        assert_eq!(
17447            error.downcast_ref::<std::io::Error>().unwrap().kind(),
17448            std::io::ErrorKind::PermissionDenied
17449        );
17450        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
17451        let mut task = crate::queue::Task::new(
17452            "review".into(),
17453            "criteria".into(),
17454            runner.state.repo.clone(),
17455            crate::queue::Source::Human,
17456        );
17457        task.record_startup_failure(failure.clone());
17458        task.record_startup_failure(failure.clone());
17459        assert_eq!(task.startup_failures.len(), 1);
17460        assert!(task.runs.is_empty());
17461        assert_eq!(task.attempts, 0);
17462    }
17463
17464    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
17465        ask_test_home();
17466        let tmp = tempfile::tempdir().unwrap();
17467        let repo = tmp.path().join("repo");
17468        std::fs::create_dir(&repo).unwrap();
17469        init_repo(&repo);
17470        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
17471        let origin = tmp.path().join("origin.git");
17472        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
17473            .await
17474            .unwrap();
17475        git::git(
17476            &repo,
17477            &["remote", "add", "origin", origin.to_str().unwrap()],
17478        )
17479        .await
17480        .unwrap();
17481        git::git(&repo, &["checkout", "-b", "fix/retained"])
17482            .await
17483            .unwrap();
17484        std::fs::write(repo.join("retained"), "retained change").unwrap();
17485        git::git(&repo, &["add", "."]).await.unwrap();
17486        git::git(&repo, &["commit", "-m", "retained implementation"])
17487            .await
17488            .unwrap();
17489        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
17490        git::git(&repo, &["checkout", "main"]).await.unwrap();
17491        std::fs::write(repo.join("base-change"), "new base").unwrap();
17492        git::git(&repo, &["add", "."]).await.unwrap();
17493        git::git(&repo, &["commit", "-m", "base moved"])
17494            .await
17495            .unwrap();
17496        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17497        git::git(&repo, &["checkout", "fix/retained"])
17498            .await
17499            .unwrap();
17500        let mut runner = runner_at(RunStatus::Voting);
17501        runner.state.repo = repo.clone();
17502        runner.state.base_commit = base;
17503        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
17504        runner.state.config.disk.min_free_bytes = 0;
17505        runner.state.config.graph.review_rounds = 1;
17506        runner.state.config.verify.e2e = vec!["true".into()];
17507        runner.state.config.verify.gate = vec!["true".into()];
17508        runner.state.candidates = vec![Candidate {
17509            index: 0,
17510            label: 'A',
17511            agent: "fixture".into(),
17512            branch: "fix/retained".into(),
17513            worktree: repo.clone(),
17514            summary: String::new(),
17515            stat: String::new(),
17516            files: 1,
17517            commits: 1,
17518            empty: false,
17519            failed: None,
17520            verified_noop: None,
17521            duration_ms: 0,
17522            folded: false,
17523        }];
17524        runner.state.tally = Some(Tally {
17525            first_choice: BTreeMap::new(),
17526            borda: BTreeMap::new(),
17527            winner: 'A',
17528            rankings: 0,
17529            unanimous_initial: true,
17530            deliberated: false,
17531            changed_votes: 0,
17532            unanimous_final: true,
17533            tie_break: None,
17534            judges: 0,
17535            present: 0,
17536            quorum: 0,
17537            met_quorum: true,
17538            uncontested: Some("review-only".into()),
17539        });
17540        let mut task = crate::queue::Task::new(
17541            "fixture".into(),
17542            "fixture".into(),
17543            repo,
17544            crate::queue::Source::Human,
17545        );
17546        task.link_run(&runner.state.id);
17547        crate::queue::Queue::open().put(&mut task).unwrap();
17548        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
17549        runner.state.base_sync = Some(BaseSync {
17550            tip: String::new(),
17551            behind: 1,
17552            attempts: BASE_SYNC_ROUNDS,
17553            conflict: None,
17554            already_in: None,
17555        });
17556        runner.sync_to_base().await.unwrap();
17557        assert_eq!(runner.state.status, RunStatus::Blocked);
17558        (tmp, runner, head, task.id)
17559    }
17560
17561    #[tokio::test]
17562    async fn rebase_preparation_failure_preserves_base_sync_budget_and_resumes() {
17563        let (_tmp, mut runner, head, _task) = exhausted_base_fixture().await;
17564        runner.state.status = RunStatus::Voting;
17565        runner.state.base_sync = None;
17566        runner.state.base_sync_recovery = Default::default();
17567        let scratch = runner.state.dir().join("base-sync");
17568        std::fs::create_dir_all(&scratch).unwrap();
17569        std::fs::write(scratch.join("obstruction"), "keep").unwrap();
17570        runner.sync_to_base().await.unwrap();
17571        assert!(runner.state.resource_block.is_some());
17572        assert!(!runner.state.base_sync_blocked());
17573        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 0);
17574        assert_eq!(runner.state.base_episode_used(), 0);
17575        assert!(runner.state.base_sync_recovery.operations.is_empty());
17576        assert!(runner.state.rebase_fixes.is_empty());
17577        assert_eq!(
17578            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17579            head
17580        );
17581        std::fs::remove_dir_all(&scratch).unwrap();
17582        runner.state.resource_block = None;
17583        runner.sync_to_base().await.unwrap();
17584        assert!(!runner.state.base_sync_blocked());
17585        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17586        assert_eq!(runner.state.base_episode_used(), 1);
17587        assert!(runner.state.base_sync_recovery.operations[0].finished);
17588        assert!(runner.state.rebase_fixes.is_empty());
17589        assert!(git::is_ancestor(&runner.state.repo, "origin/main", "HEAD").await);
17590        assert!(git::is_clean(&runner.state.repo).await.unwrap());
17591    }
17592
17593    #[cfg(unix)]
17594    #[tokio::test]
17595    async fn base_recovery_control_rechecks_owner_decision_and_routing_at_consumption() {
17596        for routed in [false, true] {
17597            let (_tmp, mut runner, head, owner) = exhausted_base_fixture().await;
17598            let home = crate::run::home();
17599            let queue = crate::queue::Queue::open();
17600            let task = queue.get(&owner).unwrap();
17601            runner.state.task_instruction = Some(task.instruction.clone());
17602            runner.state.driver_exited = true;
17603            runner.state.save().unwrap();
17604            let questions = crate::ask::Questions::at(home.join("questions"));
17605            let mut q = crate::ask::Question::new(
17606                runner.state.id.clone(),
17607                "base-recovery".into(),
17608                "owner".into(),
17609                "Recover bounded base movement?".into(),
17610                String::new(),
17611                vec!["recover".into()],
17612            );
17613            q.actions.insert(
17614                "recover".into(),
17615                crate::ask::ChoiceAction::RecoverBase {
17616                    run: runner.state.id.clone(),
17617                    stop: runner.state.base_sync_recovery.stop.clone().unwrap(),
17618                    episode: runner.state.base_sync_recovery.episodes.last().unwrap().id,
17619                },
17620            );
17621            q.routed_task = Some(owner.clone());
17622            q.answer(crate::ask::Answer::Choice("recover".into()))
17623                .unwrap();
17624            questions.put(&mut q).unwrap();
17625            crate::base_recovery_control::recover(
17626                &queue,
17627                &home,
17628                crate::base_recovery_control::Request {
17629                    run: runner.state.id.clone(),
17630                    head: head.clone(),
17631                    actor: "decisions".into(),
17632                    request: q.id.clone(),
17633                    decision: Some((owner.clone(), q.answered_at.unwrap())),
17634                    expected_stop: Some((
17635                        runner.state.base_sync_recovery.stop.clone().unwrap(),
17636                        runner.state.base_sync_recovery.episodes.last().unwrap().id,
17637                    )),
17638                },
17639            )
17640            .await
17641            .unwrap();
17642            if routed {
17643                questions
17644                    .update(&q.id, |q| {
17645                        q.routed_task = Some("another-owner".into());
17646                        Ok(())
17647                    })
17648                    .unwrap();
17649            } else {
17650                let mut task = queue.get(&owner).unwrap();
17651                task.hold_manual(Some("new owner decision".into()));
17652                // Keep the generation to exercise the separate owner-decision binding.
17653                task.status = crate::queue::TaskStatus::Queued;
17654                queue.put(&mut task).unwrap();
17655            }
17656            runner.state = RunState::load(&runner.state.id).unwrap();
17657            runner.sync_to_base().await.unwrap();
17658            let approval = runner
17659                .state
17660                .base_sync_recovery
17661                .authorizations
17662                .last()
17663                .unwrap();
17664            assert!(approval.consumed);
17665            assert!(approval.invalidated.as_ref().unwrap().contains(if routed {
17666                "routing"
17667            } else {
17668                "owner decision"
17669            }));
17670            assert_eq!(
17671                runner.state.base_sync.as_ref().unwrap().attempts,
17672                BASE_SYNC_ROUNDS
17673            );
17674            assert_eq!(
17675                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17676                head
17677            );
17678        }
17679    }
17680
17681    #[cfg(unix)]
17682    #[tokio::test]
17683    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
17684        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
17685        if std::env::var_os(CHILD).is_none() {
17686            let home = tempfile::tempdir().unwrap();
17687            let output = std::process::Command::new(std::env::current_exe().unwrap())
17688                .args([
17689                    "--exact",
17690                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
17691                    "--nocapture",
17692                ])
17693                .env(CHILD, "1")
17694                .env("MAGI_HOME", home.path())
17695                .env("MAGI_WORKER_HOME", home.path())
17696                .env("MAGI_WORKER_ID", "abcdef")
17697                .output()
17698                .unwrap();
17699            assert!(
17700                output.status.success()
17701                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17702                "{}\n{}",
17703                String::from_utf8_lossy(&output.stdout),
17704                String::from_utf8_lossy(&output.stderr)
17705            );
17706            return;
17707        }
17708        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
17709        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17710        let home = crate::run::home();
17711        let queue = crate::queue::Queue::open();
17712        let mut owner = queue.get(&task).unwrap();
17713        owner.release_operator("test-release", None);
17714        owner.start(runner.state.id.clone());
17715        queue.put(&mut owner).unwrap();
17716        runner.state.driver_exited = true;
17717        runner.state.task_instruction = Some(owner.instruction.clone());
17718        runner.state.save().unwrap();
17719        let _claim = queue.claim(&task).unwrap();
17720        let reserve = |id: &str| {
17721            let directory = home.join("workers/jobs").join(id);
17722            std::fs::create_dir_all(&directory).unwrap();
17723            let job = crate::worker::Job {
17724                id: id.into(),
17725                sequence: 1,
17726                kind: "task".into(),
17727                key: format!("task:{task}"),
17728                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
17729                generation: "fixture-generation".into(),
17730                version: None,
17731                status: crate::worker::Status::Running,
17732                result: None,
17733                pid: Some(std::process::id()),
17734                created_at: 0,
17735                finished_at: None,
17736            };
17737            std::fs::write(
17738                directory.join("job.json"),
17739                serde_json::to_vec(&job).unwrap(),
17740            )
17741            .unwrap();
17742            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
17743        };
17744        let _own = reserve("abcdef");
17745        let other = reserve("fedcba");
17746        assert!(
17747            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17748                .await
17749                .is_err()
17750        );
17751        assert!(
17752            RunState::load(&runner.state.id)
17753                .unwrap()
17754                .base_sync_recovery
17755                .authorizations
17756                .is_empty()
17757        );
17758        drop(other);
17759        assert!(crate::daemon::is_working_on(
17760            &home,
17761            &runner.state.id,
17762            Timestamp::now()
17763        ));
17764        assert!(
17765            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
17766        );
17767        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
17768            .await
17769            .unwrap();
17770        assert!(
17771            RunState::load(&runner.state.id)
17772                .unwrap()
17773                .base_recovery_pending()
17774        );
17775    }
17776
17777    #[cfg(unix)]
17778    #[tokio::test]
17779    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
17780        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17781        let home = crate::run::home();
17782        runner.state.driver_pid = Some(std::process::id());
17783        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
17784        runner.state.driver_exited = false;
17785        runner.state.begin_pass();
17786        runner.state.save().unwrap();
17787        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
17788        driver.record(&runner.state).unwrap();
17789        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
17790        // Preserve the durable claim as if the process died before Drop saved.
17791        drop(driver);
17792        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
17793        assert_eq!(
17794            runner.state.liveness_under(&home, false),
17795            crate::run::Liveness::Dead
17796        );
17797        let queue = crate::queue::Queue::open();
17798        let _claim = queue.claim(&task).unwrap();
17799        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17800            .await
17801            .unwrap();
17802        let disk = RunState::load(&runner.state.id).unwrap();
17803        assert!(!disk.driver_exited);
17804        assert!(disk.base_recovery_pending());
17805    }
17806
17807    #[cfg(unix)]
17808    #[tokio::test]
17809    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
17810        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
17811        if std::env::var_os(CHILD).is_none() {
17812            use std::os::unix::fs::PermissionsExt;
17813            let tmp = tempfile::tempdir().unwrap();
17814            let gh = tmp.path().join("gh");
17815            std::fs::write(&gh, r#"#!/bin/sh
17816case "$1 $2" in
17817  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
17818  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
17819  *) exit 2 ;;
17820esac
17821"#).unwrap();
17822            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
17823            let mut paths = vec![tmp.path().to_path_buf()];
17824            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
17825            let output = std::process::Command::new(std::env::current_exe().unwrap())
17826                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
17827                .env(CHILD, "1")
17828                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
17829                .env("PATH", std::env::join_paths(paths).unwrap())
17830                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
17831                .output().unwrap();
17832            assert!(
17833                output.status.success()
17834                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
17835                "{}\n{}",
17836                String::from_utf8_lossy(&output.stdout),
17837                String::from_utf8_lossy(&output.stderr)
17838            );
17839            return;
17840        }
17841        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17842        let repo = runner.state.repo.clone();
17843        git::git(&repo, &["checkout", "main"]).await.unwrap();
17844        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
17845        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
17846        git::git(&repo, &["checkout", "fix/retained"])
17847            .await
17848            .unwrap();
17849        runner.state.pr = Some(crate::run::PrRecord {
17850            url: "https://example.test/project/pull/1".into(),
17851            number: 1,
17852            state: "open".into(),
17853            checks: "unknown".into(),
17854            round: 0,
17855            rounds: 1,
17856            red_at_merge: Vec::new(),
17857        });
17858        runner.sync_to_base().await.unwrap();
17859        assert_eq!(runner.state.status, RunStatus::Blocked);
17860        assert_eq!(
17861            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17862            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
17863        );
17864        assert!(runner.state.base_sync_closure_pending());
17865        assert!(crate::handover::can_resume(&runner.state));
17866        let queue = crate::queue::Queue::open();
17867        assert!(!crate::base_sync_recovery::held_task(
17868            &queue,
17869            &queue.get(&task).unwrap()
17870        ));
17871        assert!(runner.state.authorize_base_sync(&task).is_err());
17872        // Legacy closure stops have no typed stop but must retain the checkpoint.
17873        runner.state.base_sync_recovery.stop = None;
17874        runner.state.status = RunStatus::Voting;
17875        runner.state.gate_ran = true;
17876        runner.state.config.graph.review_rounds = 0;
17877        runner.state.save().unwrap();
17878        assert!(crate::handover::can_resume(&runner.state));
17879        runner.state = RunState::load(&runner.state.id).unwrap();
17880        runner.execute_graph().await.unwrap();
17881        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
17882        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
17883        assert_eq!(
17884            runner.state.base_sync.as_ref().unwrap().attempts,
17885            BASE_SYNC_ROUNDS
17886        );
17887        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
17888        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
17889        assert!(runner.state.reviews.is_empty());
17890        assert!(runner.state.gate.is_empty());
17891    }
17892
17893    #[cfg(unix)]
17894    #[tokio::test]
17895    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
17896        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
17897        runner.state.driver_exited = true;
17898        runner.state.save().unwrap();
17899        let home = crate::run::home();
17900        let queue = crate::queue::Queue::open();
17901        let _claim = queue.claim(&task).unwrap();
17902        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
17903        assert!(
17904            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17905                .await
17906                .is_err()
17907        );
17908        drop(lease);
17909        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
17910        assert!(
17911            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17912                .await
17913                .is_err()
17914        );
17915        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
17916        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17917            .await
17918            .unwrap();
17919        let disk = RunState::load(&runner.state.id).unwrap();
17920        assert!(disk.base_recovery_pending());
17921        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
17922        assert!(
17923            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17924                .await
17925                .is_err()
17926        );
17927        assert_eq!(
17928            RunState::load(&runner.state.id)
17929                .unwrap()
17930                .base_sync_recovery
17931                .authorizations
17932                .len(),
17933            1
17934        );
17935        let mut owner = queue.get(&task).unwrap();
17936        owner.runs.push("20260101-000000-ffff".into());
17937        queue.put(&mut owner).unwrap();
17938        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
17939            .await
17940            .unwrap_err();
17941        assert!(error.to_string().contains("run task ownership changed"));
17942    }
17943
17944    #[cfg(unix)]
17945    #[tokio::test]
17946    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
17947        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17948        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17949        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
17950        legacy["status"] = serde_json::json!("voting");
17951        runner.state = serde_json::from_value(legacy).unwrap();
17952        runner.state.save().unwrap();
17953        for _ in 0..3 {
17954            runner.execute_graph().await.unwrap();
17955            assert_eq!(runner.state.status, RunStatus::Blocked);
17956            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17957            assert!(!crate::handover::can_resume(&runner.state));
17958            assert_eq!(
17959                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17960                head
17961            );
17962            assert!(runner.state.reviews.is_empty());
17963            assert!(!runner.state.gate_ran);
17964        }
17965        let notice = crate::notices::run_ended(&runner.state).unwrap();
17966        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
17967        runner.state.begin_pass();
17968        runner.state.status = RunStatus::Voting;
17969        runner.deliberate().await.unwrap();
17970        runner.vote().await.unwrap();
17971        runner.finish_after_tally().await.unwrap();
17972        assert_eq!(runner.state.status, RunStatus::Blocked);
17973        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
17974        assert_eq!(
17975            crate::notices::run_ended(&runner.state).unwrap().since,
17976            notice.since
17977        );
17978    }
17979
17980    #[cfg(unix)]
17981    #[tokio::test]
17982    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
17983        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17984        for detail in [
17985            "fix/retained conflicts with origin/main and did not rebase: conflict",
17986            "rebased fix/retained locally but origin refused the push",
17987            "unknown failure",
17988        ] {
17989            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
17990            runner.state.record_base_sync_stop(&head);
17991            assert!(runner.state.authorize_base_sync(&task).is_err());
17992            runner.state.status = RunStatus::Voting;
17993            runner.execute_graph().await.unwrap();
17994            assert_eq!(runner.state.status, RunStatus::Blocked);
17995            assert!(!crate::handover::can_resume(&runner.state));
17996            assert_eq!(
17997                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17998                head
17999            );
18000        }
18001    }
18002
18003    #[cfg(unix)]
18004    #[tokio::test]
18005    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
18006        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18007        runner.state.authorize_base_sync(&task).unwrap();
18008        assert!(runner.state.consume_base_recovery(&head));
18009        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18010        runner.state.reserve_base_operation(&head, &tip);
18011        runner.state.save().unwrap();
18012        runner.state = RunState::load(&runner.state.id).unwrap();
18013        assert!(!runner.state.consume_base_recovery(&head));
18014        runner.sync_to_base().await.unwrap();
18015        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18016        assert_eq!(runner.state.base_episode_used(), 1);
18017        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
18018        assert!(
18019            runner
18020                .state
18021                .base_sync_recovery
18022                .operations
18023                .last()
18024                .unwrap()
18025                .finished
18026        );
18027        assert!(!runner.state.base_sync_blocked());
18028        runner.sync_to_base().await.unwrap();
18029        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18030    }
18031
18032    #[cfg(unix)]
18033    #[tokio::test]
18034    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
18035        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18036        // Both historical diagnostic spellings retain an exhaustion identity.
18037        let original = runner
18038            .state
18039            .base_sync
18040            .as_ref()
18041            .unwrap()
18042            .conflict
18043            .clone()
18044            .unwrap();
18045        runner.state.base_sync.as_mut().unwrap().conflict = Some(
18046            original
18047                .replace("synchronization(s)", "rebase(s)")
18048                .replace("synchronizing", "rebasing"),
18049        );
18050        runner.state.base_sync_recovery = Default::default();
18051        runner.state.record_base_sync_stop(&head);
18052        assert_eq!(
18053            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
18054            crate::base_sync_recovery::StopKind::Exhausted
18055        );
18056        runner.state.authorize_base_sync(&task).unwrap();
18057        let mut legacy = serde_json::to_value(&runner.state).unwrap();
18058        let auth = legacy["base_sync_recovery"]["authorizations"][0]
18059            .as_object_mut()
18060            .unwrap();
18061        auth.remove("interval");
18062        auth.remove("scope");
18063        runner.state = serde_json::from_value(legacy).unwrap();
18064        assert!(runner.state.consume_base_recovery(&head));
18065        assert_eq!(runner.state.base_episode_used(), 3);
18066        runner.sync_to_base().await.unwrap();
18067        assert_eq!(runner.state.base_episode_used(), 4);
18068        runner.state.resource_block = Some(crate::run::ResourceBlock {
18069            node: "gate".into(),
18070            kind: "cache".into(),
18071            detail: "legacy wait".into(),
18072            at: Timestamp::now(),
18073            cause: crate::run::ResourceBlockCause::Unrecorded,
18074        });
18075        runner.state.save().unwrap();
18076        assert!(
18077            runner
18078                .state
18079                .base_sync_recovery
18080                .episodes
18081                .last()
18082                .unwrap()
18083                .boundary
18084                .is_none()
18085        );
18086        runner.state.resource_block = None;
18087        advance_base_fixture(&runner).await;
18088        runner.sync_to_base().await.unwrap();
18089        assert!(runner.state.base_sync_blocked());
18090        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18091    }
18092
18093    #[cfg(unix)]
18094    async fn advance_base_fixture(runner: &Runner) {
18095        let repo = &runner.state.repo;
18096        for args in [
18097            vec!["checkout", "main"],
18098            vec!["commit", "--allow-empty", "-m", "base advanced"],
18099            vec!["push", "origin", "main"],
18100            vec!["checkout", "fix/retained"],
18101        ] {
18102            git::git(repo, &args).await.unwrap();
18103        }
18104    }
18105
18106    #[cfg(unix)]
18107    fn fresh_base_interval_fixture(runner: &mut Runner) {
18108        runner.state.base_sync.as_mut().unwrap().attempts = 0;
18109        runner.state.base_sync.as_mut().unwrap().conflict = None;
18110        runner.state.base_sync_recovery = Default::default();
18111        runner.state.status = RunStatus::Voting;
18112    }
18113
18114    #[cfg(unix)]
18115    #[tokio::test]
18116    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
18117        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
18118        fresh_base_interval_fixture(&mut runner);
18119        runner.state.begin_pass();
18120        runner.sync_to_base().await.unwrap();
18121        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18122        runner.state.resource_block = Some(crate::run::ResourceBlock {
18123            node: "gate".into(),
18124            kind: "build cache".into(),
18125            detail: "busy".into(),
18126            at: Timestamp::now(),
18127            cause: crate::run::ResourceBlockCause::Timeout {
18128                waited_secs: 1,
18129                limit_secs: 1,
18130            },
18131        });
18132        runner.state.save().unwrap();
18133        let queue = crate::queue::Queue::open();
18134        let mut owner = queue.get(&task).unwrap();
18135        runner.state.task_instruction = Some(owner.instruction.clone());
18136        owner.release_operator("fixture-owner", None);
18137        queue.put(&mut owner).unwrap();
18138        let approval = owner.resumption.clone().unwrap();
18139        runner.state.resource_block = None;
18140        runner.state.save().unwrap();
18141        runner.state = RunState::load(&runner.state.id).unwrap();
18142        advance_base_fixture(&runner).await;
18143        runner.sync_to_base().await.unwrap();
18144        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18145        assert_eq!(runner.state.base_episode_used(), 1);
18146        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
18147        assert!(
18148            runner.state.base_sync_recovery.episodes[0]
18149                .boundary
18150                .as_ref()
18151                .unwrap()
18152                .consumed
18153        );
18154        assert!(
18155            !runner
18156                .state
18157                .renew_base_episode(&task, &approval, &head, "base")
18158        );
18159        for _ in 0..3 {
18160            owner.release_operator("fixture-owner", None);
18161            queue.put(&mut owner).unwrap();
18162            runner.state = RunState::load(&runner.state.id).unwrap();
18163            runner.state.begin_pass();
18164            runner.sync_to_base().await.unwrap();
18165            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
18166            assert_eq!(runner.state.base_episode_used(), 1);
18167        }
18168        runner.state.resource_block = Some(crate::run::ResourceBlock {
18169            node: "gate".into(),
18170            kind: "disk pressure".into(),
18171            detail: "free space required".into(),
18172            at: Timestamp::now(),
18173            cause: crate::run::ResourceBlockCause::Other,
18174        });
18175        runner.state.end_pass_parked(ParkCause::Interrupt {
18176            reason: "disk pressure".into(),
18177        });
18178        runner.state.save().unwrap();
18179        owner.release_operator("fixture-owner-after-disk", None);
18180        queue.put(&mut owner).unwrap();
18181        runner.state.resource_block = None;
18182        advance_base_fixture(&runner).await;
18183        runner.sync_to_base().await.unwrap();
18184        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
18185        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
18186        assert_eq!(runner.state.base_episode_used(), 1);
18187        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
18188    }
18189
18190    #[cfg(unix)]
18191    #[tokio::test]
18192    async fn base_recovery_continuous_advancement_stops_at_four() {
18193        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18194        fresh_base_interval_fixture(&mut runner);
18195        for spent in 1..=BASE_SYNC_ROUNDS {
18196            runner.sync_to_base().await.unwrap();
18197            assert!(!runner.state.base_sync_blocked());
18198            assert_eq!(runner.state.base_episode_used(), spent);
18199            advance_base_fixture(&runner).await;
18200        }
18201        runner.sync_to_base().await.unwrap();
18202        assert_eq!(
18203            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
18204            crate::base_sync_recovery::StopKind::Exhausted
18205        );
18206        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
18207        assert!(stop.unverified_sync);
18208        let action = runner.state.base_sync_next_action();
18209        assert!(action.contains("never reached its gate"));
18210        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
18211        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18212        runner.state = RunState::load(&runner.state.id).unwrap();
18213        runner.sync_to_base().await.unwrap();
18214        assert_eq!(runner.state.base_episode_used(), 4);
18215        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
18216    }
18217
18218    #[cfg(unix)]
18219    #[tokio::test]
18220    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
18221        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
18222        runner.state.ensure_base_episode();
18223        // Verification progress alone is not evidence of a successful sync.
18224        runner.state.record_base_progress(&head);
18225        runner.state.base_sync_recovery.stop = None;
18226        runner.state.record_base_sync_stop(&head);
18227        assert!(
18228            !runner
18229                .state
18230                .base_sync_recovery
18231                .stop
18232                .as_ref()
18233                .unwrap()
18234                .unverified_sync
18235        );
18236
18237        runner.state.finish_base_operation(Some(&head));
18238        // A gate for another head cannot verify this synchronization.
18239        runner.state.gate_ran = true;
18240        runner.state.gate_head = Some("older-head".into());
18241        runner.state.base_sync_recovery.stop = None;
18242        runner.state.record_base_sync_stop(&head);
18243        runner.state.save().unwrap();
18244        runner.state = RunState::load(&runner.state.id).unwrap();
18245        assert!(
18246            runner
18247                .state
18248                .base_sync_recovery
18249                .stop
18250                .as_ref()
18251                .unwrap()
18252                .unverified_sync
18253        );
18254
18255        runner.state.gate_head = Some(head.clone());
18256        runner.state.base_sync_recovery.stop = None;
18257        runner.state.record_base_sync_stop(&head);
18258        assert!(
18259            !runner
18260                .state
18261                .base_sync_recovery
18262                .stop
18263                .as_ref()
18264                .unwrap()
18265                .unverified_sync
18266        );
18267        assert!(
18268            !runner
18269                .state
18270                .base_sync_next_action()
18271                .contains("never reached its gate")
18272        );
18273
18274        // Historical records cannot establish the missing synchronization evidence.
18275        let mut legacy = serde_json::to_value(&runner.state).unwrap();
18276        legacy["base_sync_recovery"]["stop"]
18277            .as_object_mut()
18278            .unwrap()
18279            .remove("unverified_sync");
18280        legacy["base_sync_recovery"]["episodes"][0]
18281            .as_object_mut()
18282            .unwrap()
18283            .remove("last_sync_head");
18284        runner.state = serde_json::from_value(legacy).unwrap();
18285        assert!(
18286            !runner
18287                .state
18288                .base_sync_recovery
18289                .stop
18290                .as_ref()
18291                .unwrap()
18292                .unverified_sync
18293        );
18294        assert!(
18295            runner.state.base_sync_recovery.episodes[0]
18296                .last_sync_head
18297                .is_none()
18298        );
18299    }
18300
18301    #[cfg(unix)]
18302    #[tokio::test]
18303    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
18304        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18305        runner.state.authorize_base_sync(&task).unwrap();
18306        assert!(
18307            runner
18308                .state
18309                .base_sync_next_action()
18310                .contains("Approval recorded")
18311        );
18312        assert!(
18313            runner
18314                .state
18315                .base_sync_next_action()
18316                .contains("4/4 in episode 0, 4 lifetime")
18317        );
18318        assert!(runner.state.consume_base_recovery(&head));
18319        assert!(
18320            runner
18321                .state
18322                .base_sync_next_action()
18323                .contains("0/4 in episode 1, 4 lifetime")
18324        );
18325    }
18326
18327    #[cfg(unix)]
18328    #[tokio::test]
18329    async fn base_recovery_changed_head_invalidates_without_spending() {
18330        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
18331        runner.state.authorize_base_sync(&task).unwrap();
18332        git::git(
18333            &runner.state.repo,
18334            &["commit", "--allow-empty", "-m", "external head change"],
18335        )
18336        .await
18337        .unwrap();
18338        runner.sync_to_base().await.unwrap();
18339        let a = runner
18340            .state
18341            .base_sync_recovery
18342            .authorizations
18343            .last()
18344            .unwrap();
18345        assert!(a.consumed);
18346        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
18347        runner.state = RunState::load(&runner.state.id).unwrap();
18348        assert!(
18349            runner
18350                .state
18351                .base_sync_next_action()
18352                .contains("Approval invalidated")
18353        );
18354        assert!(!runner.state.base_recovery_pending());
18355        assert!(runner.state.base_sync_blocked());
18356        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18357    }
18358
18359    #[cfg(unix)]
18360    #[tokio::test]
18361    async fn base_recovery_stale_fetched_base_invalidates_approval() {
18362        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18363        runner.state.driver_exited = true;
18364        runner.state.save().unwrap();
18365        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18366            .await
18367            .unwrap();
18368        runner.state = RunState::load(&runner.state.id).unwrap();
18369        advance_base_fixture(&runner).await;
18370        runner.sync_to_base().await.unwrap();
18371        let a = runner
18372            .state
18373            .base_sync_recovery
18374            .authorizations
18375            .last()
18376            .unwrap();
18377        assert!(
18378            a.consumed
18379                && a.invalidated
18380                    .as_deref()
18381                    .unwrap()
18382                    .contains("fetched base changed")
18383        );
18384        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
18385        assert_eq!(
18386            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18387            head
18388        );
18389        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
18390            .await
18391            .unwrap();
18392        runner.state = RunState::load(&runner.state.id).unwrap();
18393        runner.sync_to_base().await.unwrap();
18394        assert!(!runner.state.base_sync_blocked());
18395        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18396    }
18397
18398    #[cfg(unix)]
18399    #[tokio::test]
18400    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
18401        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18402        runner.state.authorize_base_sync(&task).unwrap();
18403        assert!(runner.state.consume_base_recovery(&head));
18404        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18405        runner.state.reserve_base_operation(&head, &tip);
18406        runner.state.save().unwrap();
18407        let scratch = runner.state.dir().join("base-sync");
18408        assert_eq!(
18409            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18410                .await
18411                .unwrap(),
18412            crate::rebase::Rebased::Applied
18413        );
18414        runner.state = RunState::load(&runner.state.id).unwrap();
18415        runner.sync_to_base().await.unwrap();
18416        assert!(
18417            runner
18418                .state
18419                .base_sync_recovery
18420                .operations
18421                .last()
18422                .unwrap()
18423                .finished
18424        );
18425        assert_eq!(runner.state.base_episode_used(), 1);
18426        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18427        assert!(!runner.state.base_sync_blocked());
18428    }
18429
18430    #[cfg(unix)]
18431    #[tokio::test]
18432    async fn base_recovery_crash_preserves_untracked_files() {
18433        for ignored in [false, true] {
18434            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18435            runner.state.authorize_base_sync(&task).unwrap();
18436            assert!(runner.state.consume_base_recovery(&head));
18437            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18438            runner.state.reserve_base_operation(&head, &tip);
18439            runner.state.save().unwrap();
18440            let scratch = runner.state.dir().join("base-sync");
18441            assert_eq!(
18442                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18443                    .await
18444                    .unwrap(),
18445                crate::rebase::Rebased::Applied
18446            );
18447            let repo = &runner.state.repo;
18448            let result = git::rev_parse(repo, "HEAD").await.unwrap();
18449            assert_ne!(result, head);
18450            if ignored {
18451                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
18452                    .await
18453                    .unwrap();
18454                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
18455            }
18456            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
18457            assert!(
18458                git::git_raw(repo, &["diff", "--quiet", &head])
18459                    .await
18460                    .unwrap()
18461                    .ok()
18462            );
18463            // Resume after the branch moved, before its checkout was reset.
18464            runner.state = RunState::load(&runner.state.id).unwrap();
18465            runner.sync_to_base().await.unwrap();
18466            assert!(runner.state.base_sync_blocked());
18467            assert!(!runner.state.base_sync_recovery.operations[0].finished);
18468            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18469            assert_eq!(
18470                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18471                result
18472            );
18473            assert_eq!(
18474                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
18475                "retained work\n"
18476            );
18477            assert!(
18478                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
18479                    .await
18480                    .unwrap()
18481                    .ok()
18482            );
18483        }
18484    }
18485
18486    #[cfg(unix)]
18487    #[tokio::test]
18488    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
18489        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18490        runner.state.authorize_base_sync(&task).unwrap();
18491        assert!(runner.state.consume_base_recovery(&head));
18492        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
18493        runner.state.reserve_base_operation(&head, &tip);
18494        runner.state.save().unwrap();
18495        let scratch = runner.state.dir().join("base-sync");
18496        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
18497            .await
18498            .unwrap();
18499        git::sync_to_head(&runner.state.repo).await.unwrap();
18500        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
18501        git::git(&runner.state.repo, &["add", "foreign"])
18502            .await
18503            .unwrap();
18504        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
18505            .await
18506            .unwrap();
18507        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18508        runner.state = RunState::load(&runner.state.id).unwrap();
18509        runner.sync_to_base().await.unwrap();
18510        assert!(runner.state.base_sync_blocked());
18511        assert!(
18512            !runner
18513                .state
18514                .base_sync_recovery
18515                .operations
18516                .last()
18517                .unwrap()
18518                .finished
18519        );
18520        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18521        assert_eq!(
18522            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18523            foreign_head
18524        );
18525    }
18526
18527    #[cfg(unix)]
18528    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
18529    {
18530        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18531        fresh_base_interval_fixture(&mut runner);
18532        let repo = &runner.state.repo;
18533        git::git(repo, &["checkout", "main"]).await.unwrap();
18534        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
18535        git::git(repo, &["add", "retained"]).await.unwrap();
18536        git::git(repo, &["commit", "-m", "upstream retained change"])
18537            .await
18538            .unwrap();
18539        git::git(repo, &["push", "origin", "main"]).await.unwrap();
18540        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
18541        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
18542        let scratch = runner.state.dir().join("base-sync");
18543        assert!(matches!(
18544            git::rebase_start(repo, &scratch, "fix/retained", &tip)
18545                .await
18546                .unwrap(),
18547            git::RebaseStart::Conflicted(_)
18548        ));
18549        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
18550            agent: "fixture".into(),
18551            paths: vec!["retained".into()],
18552            from: Some(head.clone()),
18553            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
18554            index_outside: None,
18555            finished: false,
18556            error: None,
18557            launcher: None,
18558            infra: None,
18559            outage: None,
18560        });
18561        (tmp, runner, head, tip, scratch)
18562    }
18563
18564    #[cfg(unix)]
18565    #[tokio::test]
18566    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
18567        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18568        // A resource stop from before reservations existed, with resolved
18569        // contents left for the launcher's ordinary staging continuation.
18570        std::fs::write(
18571            scratch.join("retained"),
18572            "upstream and retained resolution\n",
18573        )
18574        .unwrap();
18575        runner.state.resource_block = Some(crate::run::ResourceBlock {
18576            node: "rebase".into(),
18577            kind: "repository coordination".into(),
18578            detail: "busy".into(),
18579            at: Timestamp::now(),
18580            cause: crate::run::ResourceBlockCause::Other,
18581        });
18582        runner.state.save().unwrap();
18583        // Advance the remote without moving either checkout participating
18584        // in the standing rebase.
18585        let newer = git::git(
18586            &runner.state.repo,
18587            &[
18588                "commit-tree",
18589                "main^{tree}",
18590                "-p",
18591                "main",
18592                "-m",
18593                "base advanced during wait",
18594            ],
18595        )
18596        .await
18597        .unwrap();
18598        git::git(
18599            &runner.state.repo,
18600            &[
18601                "push",
18602                "origin",
18603                &format!("{}:refs/heads/main", newer.trim()),
18604            ],
18605        )
18606        .await
18607        .unwrap();
18608        runner.state = RunState::load(&runner.state.id).unwrap();
18609        runner.state.resource_block = None;
18610        runner.sync_to_base().await.unwrap();
18611        assert!(!runner.state.base_sync_blocked());
18612        let op = &runner.state.base_sync_recovery.operations[0];
18613        assert_eq!(op.head, head);
18614        assert_eq!(op.tip, tip);
18615        assert!(op.finished);
18616        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18617        assert_eq!(runner.state.rebase_fixes.len(), 1);
18618        let latest = git::rev_parse(&runner.state.repo, "origin/main")
18619            .await
18620            .unwrap();
18621        assert_ne!(latest, tip);
18622        runner.state = RunState::load(&runner.state.id).unwrap();
18623        runner.sync_to_base().await.unwrap();
18624        assert!(!runner.state.base_sync_blocked());
18625        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
18626        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
18627        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
18628    }
18629
18630    #[cfg(unix)]
18631    #[tokio::test]
18632    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
18633        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18634        runner.state.reserve_base_operation(&head, &tip);
18635        runner.state.save().unwrap();
18636        std::fs::write(
18637            scratch.join("retained"),
18638            "upstream and retained resolution\n",
18639        )
18640        .unwrap();
18641        git::git(&scratch, &["add", "retained"]).await.unwrap();
18642        git::git(
18643            &scratch,
18644            &["-c", "core.editor=true", "rebase", "--continue"],
18645        )
18646        .await
18647        .unwrap();
18648        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18649        assert!(
18650            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18651                .await
18652                .unwrap()
18653                .is_empty()
18654        );
18655        assert_eq!(
18656            crate::rebase::finish(
18657                &mut runner.state,
18658                &scratch,
18659                "fix/retained",
18660                &head,
18661                &tip,
18662                &["retained".into()],
18663                ""
18664            )
18665            .await
18666            .unwrap(),
18667            crate::rebase::Rebased::Applied
18668        );
18669        assert!(!scratch.exists());
18670        // Simulate losing the caller before finish_base_operation/save.
18671        runner.state = RunState::load(&runner.state.id).unwrap();
18672        let op = &runner.state.base_sync_recovery.operations[0];
18673        assert!(!op.finished);
18674        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
18675        runner.sync_to_base().await.unwrap();
18676        assert!(!runner.state.base_sync_blocked());
18677        assert!(runner.state.base_sync_recovery.operations[0].finished);
18678        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18679        assert_eq!(runner.state.rebase_fixes.len(), 1);
18680        let winner = runner.state.winner().unwrap().clone();
18681        let wait = runner.repository_wait();
18682        assert!(
18683            verify_current_head(&mut runner.state, &winner, &wait)
18684                .await
18685                .unwrap()
18686        );
18687        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18688    }
18689
18690    #[cfg(unix)]
18691    #[tokio::test]
18692    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
18693        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
18694            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
18695            runner.state.reserve_base_operation(&head, &tip);
18696            runner.state.save().unwrap();
18697            std::fs::write(
18698                scratch.join("retained"),
18699                "upstream and retained resolution\n",
18700            )
18701            .unwrap();
18702            git::git(&scratch, &["add", "retained"]).await.unwrap();
18703            git::git(
18704                &scratch,
18705                &["-c", "core.editor=true", "rebase", "--continue"],
18706            )
18707            .await
18708            .unwrap();
18709            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
18710            assert!(!git::rebase_in_progress(&scratch).await);
18711            assert!(
18712                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
18713                    .await
18714                    .unwrap()
18715                    .is_empty()
18716            );
18717            match damage {
18718                Some("record") => {
18719                    let path =
18720                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
18721                            .await
18722                            .unwrap();
18723                    std::fs::write(scratch.join(path), "").unwrap();
18724                }
18725                Some("foreign") => {
18726                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
18727                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
18728                    git::git(&scratch, &["commit", "-m", "foreign work"])
18729                        .await
18730                        .unwrap();
18731                }
18732                Some("dirty") => {
18733                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
18734                }
18735                _ => {}
18736            }
18737            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18738            // Crash immediately after Git continuation, before finish runs.
18739            runner.state = RunState::load(&runner.state.id).unwrap();
18740            assert!(
18741                runner.state.base_sync_recovery.operations[0]
18742                    .validated_head
18743                    .is_none()
18744            );
18745            runner.sync_to_base().await.unwrap();
18746            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18747            assert_eq!(runner.state.base_episode_used(), 1);
18748            assert_eq!(runner.state.rebase_fixes.len(), 1);
18749            assert_eq!(
18750                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18751                retained_head
18752            );
18753            if damage.is_some() {
18754                assert!(runner.state.base_sync_blocked());
18755                assert!(scratch.exists());
18756                assert!(!runner.state.base_sync_recovery.operations[0].finished);
18757                assert!(
18758                    runner.state.base_sync_recovery.operations[0]
18759                        .validated_head
18760                        .is_none()
18761                );
18762            } else {
18763                assert!(!runner.state.base_sync_blocked());
18764                assert!(!scratch.exists());
18765                assert!(runner.state.base_sync_recovery.operations[0].finished);
18766                assert_eq!(
18767                    runner.state.base_sync_recovery.operations[0]
18768                        .validated_head
18769                        .as_deref(),
18770                    Some(result.as_str())
18771                );
18772                let winner = runner.state.winner().unwrap().clone();
18773                let wait = runner.repository_wait();
18774                assert!(
18775                    verify_current_head(&mut runner.state, &winner, &wait)
18776                        .await
18777                        .unwrap()
18778                );
18779                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
18780                runner.state = RunState::load(&runner.state.id).unwrap();
18781                runner.sync_to_base().await.unwrap();
18782                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
18783            }
18784        }
18785    }
18786
18787    #[cfg(unix)]
18788    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
18789        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
18790        // This is an ordinary gate stop, not an authorized base-budget recovery.
18791        runner.state.prep_complete = Some(true);
18792        runner.state.base_sync = None;
18793        runner.state.base_sync_recovery = Default::default();
18794        runner.state.config.graph.gate_fix_rounds = 1;
18795        let mut review = review_round(true, 0, 1, 1, false, true);
18796        review.head = head.clone();
18797        review.verified_head = Some(head.clone());
18798        runner.state.reviews = vec![review];
18799        runner.state.gate = vec![CommandOutcome {
18800            command: "old gate".into(),
18801            code: Some(1),
18802            output_tail: "old failure".into(),
18803            duration_ms: 0,
18804            resource_blocked: false,
18805        }];
18806        runner.state.gate_head = Some(head.clone());
18807        runner.state.gate_ran = true;
18808        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
18809            agent: "fixture".into(),
18810            failed: runner.state.gate.clone(),
18811            notes: "spent".into(),
18812            committed: false,
18813            error: None,
18814        }];
18815        runner.state.land_approval = Some(crate::run::LandApproval {
18816            question: "old approval".into(),
18817            head: head.clone(),
18818        });
18819        runner.state.status = RunStatus::Blocked;
18820        (tmp, runner, head)
18821    }
18822
18823    #[cfg(unix)]
18824    #[tokio::test]
18825    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
18826        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
18827            let (tmp, mut runner, old) = resumed_gate_fixture().await;
18828            let phases = tmp.path().join("phases");
18829            runner.state.config.verify.e2e = vec![format!(
18830                "echo e2e >> '{}'; {}",
18831                phases.display(),
18832                if e2e_ok {
18833                    "true"
18834                } else {
18835                    "echo new-e2e-failure; false"
18836                }
18837            )];
18838            runner.state.config.verify.gate = vec![format!(
18839                "echo gate >> '{}'; {}",
18840                phases.display(),
18841                if gate_ok {
18842                    "true"
18843                } else {
18844                    "echo new-gate-failure; false"
18845                }
18846            )];
18847            runner.state.save().unwrap();
18848            runner.state = RunState::load(&runner.state.id).unwrap();
18849            runner.sync_to_base().await.unwrap();
18850            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18851            assert_ne!(head, old);
18852            // Reentry after Git updated but before verification was invalidated.
18853            runner.state = RunState::load(&runner.state.id).unwrap();
18854            runner.execute_graph().await.unwrap();
18855            assert_eq!(runner.state.gate_fixes.len(), 1);
18856            assert_eq!(runner.state.reviews.len(), 1);
18857            assert_eq!(runner.state.reviews[0].head, old);
18858            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
18859            assert_eq!(runner.state.verification_history.len(), 1);
18860            let history = &runner.state.verification_history[0];
18861            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
18862            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18863            assert!(history.archived_at.is_some());
18864            assert_eq!(history.fixes_spent, Some(1));
18865            assert_eq!(history.gate[0].output_tail, "old failure");
18866            let archive_events: Vec<_> = runner
18867                .state
18868                .events
18869                .iter()
18870                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
18871                .collect();
18872            assert_eq!(archive_events.len(), 1);
18873            assert!(archive_events[0].message.contains(&short(&old)));
18874            assert!(archive_events[0].message.contains(&short(&head)));
18875            assert!(
18876                archive_events[0]
18877                    .message
18878                    .contains("1 gate-fix round(s) already spent stay spent")
18879            );
18880            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
18881            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
18882            if e2e_ok {
18883                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18884                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
18885            } else {
18886                assert!(!runner.state.gate_ran);
18887            }
18888            assert_eq!(
18889                runner.state.status,
18890                if e2e_ok && gate_ok {
18891                    RunStatus::Ready
18892                } else {
18893                    RunStatus::Blocked
18894                }
18895            );
18896            if !e2e_ok || !gate_ok {
18897                let cause = runner.state.verification_blocking_cause().unwrap();
18898                assert!(cause.contains(&head));
18899                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
18900                // A later successful progress event must not become the cause.
18901                runner.state.event("land", "rebased successfully");
18902                let notice = crate::notices::run_ended(&runner.state).unwrap();
18903                assert!(!format!("{notice:?}").contains("rebased successfully"));
18904                assert!(runner.state.merge.is_none());
18905            }
18906            let before = std::fs::read_to_string(&phases).unwrap();
18907            runner.state.save().unwrap();
18908            runner.state = RunState::load(&runner.state.id).unwrap();
18909            runner.review_loop().await.unwrap();
18910            runner.gate().await.unwrap();
18911            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
18912            assert_eq!(runner.state.verification_history.len(), 1);
18913            let history = &runner.state.verification_history[0];
18914            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
18915            assert!(history.archived_at.is_some());
18916            assert_eq!(history.fixes_spent, Some(1));
18917        }
18918    }
18919
18920    #[cfg(unix)]
18921    #[tokio::test]
18922    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
18923        let (tmp, mut runner, head) = resumed_gate_fixture().await;
18924        let marker = tmp.path().join("gate-ran");
18925        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
18926        runner.review_loop().await.unwrap();
18927        runner.gate().await.unwrap();
18928        assert_eq!(runner.state.status, RunStatus::Blocked);
18929        assert!(!marker.exists());
18930        assert!(runner.state.verification_history.is_empty());
18931        runner.state.gate_head = None;
18932        runner.gate().await.unwrap();
18933        assert!(marker.exists());
18934        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18935        assert_eq!(runner.state.verification_history.len(), 1);
18936        assert_eq!(runner.state.gate_fixes.len(), 1);
18937    }
18938
18939    #[cfg(unix)]
18940    #[tokio::test]
18941    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
18942        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18943        runner.sync_to_base().await.unwrap();
18944        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18945        runner.state.reconcile_verification_head(&head);
18946        // Both crash-before-command and resource wait are unresolved evidence.
18947        for outcomes in [
18948            Vec::new(),
18949            vec![CommandOutcome {
18950                command: "true".into(),
18951                code: None,
18952                output_tail: "cache unavailable".into(),
18953                duration_ms: 0,
18954                resource_blocked: true,
18955            }],
18956        ] {
18957            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18958                head: head.clone(),
18959                passed: false,
18960                outcomes,
18961            });
18962            runner.state.gate_ran = false;
18963            runner.state.gate.clear();
18964            runner.state.save().unwrap();
18965            runner.state = RunState::load(&runner.state.id).unwrap();
18966            runner.merge().await.unwrap();
18967            assert!(runner.state.merge.is_none());
18968            runner.review_loop().await.unwrap();
18969            runner.gate().await.unwrap();
18970            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18971            assert!(runner.state.gate_allows_progress());
18972            assert_eq!(runner.state.gate_fixes.len(), 1);
18973            assert!(runner.state.merge.is_none());
18974        }
18975        // Moving the winner after a successful gate cannot publish an unchecked SHA.
18976        git::git(
18977            &runner.state.repo,
18978            &["commit", "--allow-empty", "-m", "external update"],
18979        )
18980        .await
18981        .unwrap();
18982        runner.merge().await.unwrap();
18983        assert!(runner.state.merge.is_none());
18984        assert_eq!(runner.state.status, RunStatus::Blocked);
18985    }
18986
18987    #[cfg(unix)]
18988    #[tokio::test]
18989    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
18990        for waiting_on_gate in [false, true] {
18991            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
18992            runner.sync_to_base().await.unwrap();
18993            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18994            runner.state.reconcile_verification_head(&head);
18995            if waiting_on_gate {
18996                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
18997                    head: head.clone(),
18998                    passed: true,
18999                    outcomes: Vec::new(),
19000                });
19001            }
19002            let cache = tmp.path().join("cache");
19003            runner.state.config.verify.e2e =
19004                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
19005            runner.state.config.verify.gate = vec!["true".into()];
19006            runner.state.config.graph.timeout_resource_wait = 1;
19007            let owner =
19008                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
19009            let guard =
19010                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
19011                    crate::cache::AcquireOutcome::Acquired(g) => g,
19012                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
19013                };
19014            runner.review_loop().await.unwrap();
19015            runner.gate().await.unwrap();
19016            assert!(!runner.state.gate_ran);
19017            assert_eq!(runner.state.gate_fixes.len(), 1);
19018            assert!(runner.state.resource_block.is_some());
19019            runner.merge().await.unwrap();
19020            assert!(runner.state.merge.is_none());
19021            drop(guard);
19022            runner.state = RunState::load(&runner.state.id).unwrap();
19023            // The driver clears the previous pass's resource stop on resume.
19024            runner.state.resource_block = None;
19025            runner.review_loop().await.unwrap();
19026            runner.gate().await.unwrap();
19027            assert!(runner.state.gate_allows_progress());
19028            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
19029            assert_eq!(runner.state.gate_fixes.len(), 1);
19030        }
19031    }
19032
19033    #[cfg(unix)]
19034    #[tokio::test]
19035    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
19036        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
19037        runner.sync_to_base().await.unwrap();
19038        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
19039        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
19040        // Verification passes before pre_gate and fails on the mechanically changed head.
19041        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
19042        runner.review_loop().await.unwrap();
19043        runner.gate().await.unwrap();
19044        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
19045        assert_ne!(before, after);
19046        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
19047        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
19048        assert!(!runner.state.gate_allows_progress());
19049        runner.merge().await.unwrap();
19050        assert!(runner.state.merge.is_none());
19051        assert!(
19052            runner
19053                .state
19054                .verification_history
19055                .iter()
19056                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
19057        );
19058
19059        // An incomplete review is still an independent hold on a changed head.
19060        runner.state.reviews[0].clean = false;
19061        runner.state.reviews[0].answered = 0;
19062        runner.state.config.verify.e2e = vec!["true".into()];
19063        runner.review_loop().await.unwrap();
19064        runner.gate().await.unwrap();
19065        runner.merge().await.unwrap();
19066        assert_eq!(runner.state.status, RunStatus::Blocked);
19067        assert!(runner.state.merge.is_none());
19068        assert!(
19069            runner
19070                .state
19071                .verification_blocking_cause()
19072                .unwrap()
19073                .contains("review blocked")
19074        );
19075    }
19076
19077    #[cfg(unix)]
19078    #[tokio::test]
19079    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
19080        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
19081        runner.state.gate_head = None;
19082        runner.state.config.verify.gate =
19083            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
19084        runner.review_loop().await.unwrap();
19085        runner.gate().await.unwrap();
19086        assert_ne!(
19087            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
19088            old
19089        );
19090        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
19091        assert!(!runner.state.gate_allows_progress());
19092        assert_eq!(runner.state.status, RunStatus::Blocked);
19093        runner.merge().await.unwrap();
19094        assert!(runner.state.merge.is_none());
19095    }
19096
19097    #[cfg(unix)]
19098    #[tokio::test]
19099    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
19100        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
19101        let mut old_review = review_round(true, 0, 1, 1, false, true);
19102        old_review.head = head.clone();
19103        old_review.verified_head = Some(head.clone());
19104        runner.state.reviews = vec![old_review];
19105        runner.state.gate = vec![CommandOutcome {
19106            command: "old gate".into(),
19107            code: None,
19108            output_tail: "timed out after 2400s".into(),
19109            duration_ms: 2_400_000,
19110            resource_blocked: false,
19111        }];
19112        runner.state.gate_ran = true;
19113        runner.state.gate_head = Some(head.clone());
19114        runner.state.land_approval = Some(crate::run::LandApproval {
19115            question: "old-head-approval".into(),
19116            head: head.clone(),
19117        });
19118        runner.state.authorize_base_sync(&task).unwrap();
19119        runner.sync_to_base().await.unwrap();
19120        assert!(!runner.state.base_sync_blocked());
19121        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
19122        assert_ne!(head, fresh);
19123        assert_eq!(runner.state.reviews.len(), 1);
19124        assert!(!runner.state.gate_ran);
19125        let saved = &runner.state.base_sync_recovery.authorizations[0];
19126        assert_eq!(saved.reviews[0].head, head);
19127        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
19128        assert!(saved.gate_ran);
19129        assert!(!saved.gate[0].ok());
19130        runner.review_loop().await.unwrap();
19131        runner.gate().await.unwrap();
19132        assert_eq!(runner.state.reviews[0].head, head);
19133        assert_eq!(
19134            runner.state.reviews[0].verified_head.as_deref(),
19135            Some(head.as_str())
19136        );
19137        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
19138        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
19139        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
19140        assert!(runner.state.gate_ran);
19141        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
19142        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
19143        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
19144        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
19145        assert_eq!(
19146            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
19147            "retained change"
19148        );
19149    }
19150
19151    #[cfg(unix)]
19152    #[tokio::test]
19153    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
19154        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
19155        let queue = crate::queue::Queue::open();
19156        let mut owner = queue.get(&task).unwrap();
19157        owner.release_operator("test-resume", None);
19158        queue.put(&mut owner).unwrap();
19159        let generation = owner.resumption.as_ref().unwrap().generation;
19160        runner.state.task_instruction = Some(owner.instruction.clone());
19161        runner.state.driver_exited = true;
19162        runner.state.save().unwrap();
19163        let home = crate::run::home();
19164        let _claim = queue.claim(&task).unwrap();
19165        owner.resumption.as_mut().unwrap().at =
19166            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
19167        queue.put(&mut owner).unwrap();
19168        assert!(
19169            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19170                .await
19171                .is_err(),
19172            "an approval from before the stop grants nothing"
19173        );
19174        owner.resumption.as_mut().unwrap().at = Timestamp::now();
19175        queue.put(&mut owner).unwrap();
19176        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19177            .await
19178            .unwrap();
19179        assert!(
19180            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19181                .await
19182                .is_err()
19183        );
19184        runner.state = RunState::load(&runner.state.id).unwrap();
19185        assert_eq!(
19186            runner.state.base_sync_recovery.authorizations[0].generation,
19187            Some(generation)
19188        );
19189        runner.sync_to_base().await.unwrap();
19190        let repo = runner.state.repo.clone();
19191        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
19192        git::git(&repo, &["checkout", "main"]).await.unwrap();
19193        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
19194        git::git(&repo, &["add", "."]).await.unwrap();
19195        git::git(&repo, &["commit", "-m", "move base again"])
19196            .await
19197            .unwrap();
19198        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
19199        git::git(&repo, &["checkout", "fix/retained"])
19200            .await
19201            .unwrap();
19202        runner.sync_to_base().await.unwrap();
19203        assert!(!runner.state.base_sync_blocked());
19204        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
19205        assert_eq!(runner.state.base_episode_used(), 2);
19206        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
19207        assert_ne!(fresh, head);
19208        assert!(!runner.state.base_recovery_pending());
19209        // Even replaying the generation with a later timestamp grants nothing.
19210        owner.resumption.as_mut().unwrap().at = Timestamp::now();
19211        queue.put(&mut owner).unwrap();
19212        assert!(
19213            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
19214                .await
19215                .is_err()
19216        );
19217        runner.state = RunState::load(&runner.state.id).unwrap();
19218        runner.sync_to_base().await.unwrap();
19219        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
19220        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
19221        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
19222    }
19223
19224    #[cfg(unix)]
19225    #[tokio::test]
19226    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
19227        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
19228        runner.state.base_sync = None;
19229        runner.state.status = RunStatus::Blocked;
19230        let mut second = runner.state.candidates[0].clone();
19231        second.label = 'B';
19232        runner.state.candidates.push(second);
19233        assert!(runner.state.votes.is_empty());
19234        assert!(runner.state.deliberation.is_empty());
19235        runner.deliberate().await.unwrap();
19236        runner.vote().await.unwrap();
19237        assert_eq!(runner.state.status, RunStatus::Blocked);
19238        assert!(runner.state.votes.is_empty());
19239        assert!(runner.state.deliberation.is_empty());
19240    }
19241
19242    /// A minimal, git-free `Runner` at a given status — `settle_questions`
19243    /// reads nothing else off it.
19244    fn runner_at(status: RunStatus) -> Runner {
19245        let mut state = RunState::new(
19246            PathBuf::from("/nonexistent/repo"),
19247            "main".to_owned(),
19248            "deadbeef".to_owned(),
19249            "task".to_owned(),
19250            Config::default(),
19251        );
19252        // These fixtures test execution flow, independent of host disk capacity.
19253        state.config.disk.min_free_bytes = 0;
19254        // These fixtures share one home and save concurrently. Short clock
19255        // IDs can collide, making an ownership test read another test's run.
19256        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
19257        state.id = format!(
19258            "20000101-000000-{:04x}",
19259            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
19260        );
19261        state.status = status;
19262        Runner {
19263            state,
19264            roles: ResolvedRoles {
19265                implementers: Vec::new(),
19266                judges: Vec::new(),
19267                reviewers: Vec::new(),
19268                fixer: None,
19269                conductor: conductor(),
19270                implementer_roster: Vec::new(),
19271                judge_roster: Vec::new(),
19272                reviewer_roster: Vec::new(),
19273            },
19274            sem: Arc::new(Semaphore::new(1)),
19275            pause: Pause::new(),
19276            interrupt: Pause::new(),
19277        }
19278    }
19279
19280    #[tokio::test]
19281    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
19282        crate::run::pin_test_home();
19283        let temp = tempfile::tempdir().unwrap();
19284        let root = temp.path().canonicalize().unwrap();
19285        let repo = root.join("repo");
19286        std::fs::create_dir(&repo).unwrap();
19287        init_repo(&repo);
19288        git::git(&repo, &["config", "core.autocrlf", "false"])
19289            .await
19290            .unwrap();
19291        let mut runner = runner_at(RunStatus::Blocked);
19292        runner.state.repo = repo.clone();
19293        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
19294        runner.state.config.agents = vec![conductor()];
19295        runner.state.config.graph.selected_implementer = Some("conductor".into());
19296        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
19297        let (shell, command, setup_output) = if cfg!(windows) {
19298            (
19299                vec!["cmd".into(), "/C".into()],
19300                "echo ready>seat-ready",
19301                "ready\r\n",
19302            )
19303        } else {
19304            (
19305                vec!["sh".into(), "-c".into()],
19306                "printf ready > seat-ready",
19307                "ready",
19308            )
19309        };
19310        runner.state.config.verify.shell = Some(shell);
19311        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
19312            run: Some(command.into()),
19313            ..Default::default()
19314        }];
19315        runner.state.save().unwrap();
19316        let path = runner.state.worktree_root().join("judge-1");
19317        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
19318            .await
19319            .unwrap();
19320        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
19321        git::git(&path, &["add", "."]).await.unwrap();
19322        let tree = git::git(&path, &["write-tree"]).await.unwrap();
19323        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
19324            .await
19325            .unwrap();
19326        git::git(&path, &["checkout", "--detach", &head])
19327            .await
19328            .unwrap();
19329        let evidence = crate::fold_evidence::snapshot(
19330            &runner.state,
19331            &crate::run::home().canonicalize().unwrap(),
19332            &head,
19333            64 * 1024 * 1024,
19334        )
19335        .await
19336        .unwrap();
19337        let mut contents = BTreeMap::new();
19338        contents.insert(
19339            repo.join("logs/HEAD"),
19340            format!("{head} {head} fixture\n").into_bytes(),
19341        );
19342        let record = serde_json::json!({
19343            "version":1, "run":runner.state.id, "source":path, "head":head,
19344            "evidence":evidence, "admin":{}, "admin_contents":contents,
19345            "history":{head.clone():evidence}, "phase":"evicted"
19346        });
19347        std::fs::write(
19348            runner.state.dir().join("artifacts/eviction-judge-1.json"),
19349            serde_json::to_vec(&record).unwrap(),
19350        )
19351        .unwrap();
19352        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
19353        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
19354            .await
19355            .unwrap();
19356        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
19357        assert!(
19358            !git::git_raw(&repo, &["cat-file", "-e", &head])
19359                .await
19360                .unwrap()
19361                .ok()
19362        );
19363        let mut resumed = Runner::resume(&runner.state.id).unwrap();
19364        let error = resumed.execute().await.unwrap_err();
19365        assert!(
19366            error
19367                .to_string()
19368                .contains("selected implementation is held"),
19369            "{error:#}"
19370        );
19371        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
19372        assert_eq!(
19373            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
19374            setup_output
19375        );
19376        assert_eq!(
19377            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
19378            "unrelated seat"
19379        );
19380        let error = resumed.execute().await.unwrap_err();
19381        assert!(
19382            error
19383                .to_string()
19384                .contains("selected implementation is held"),
19385            "{error:#}"
19386        );
19387        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
19388        git::git(&path, &["add", "."]).await.unwrap();
19389        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
19390        let next = git::git(
19391            &path,
19392            &[
19393                "commit-tree",
19394                &next_tree,
19395                "-p",
19396                &head,
19397                "-m",
19398                "next review checkpoint",
19399            ],
19400        )
19401        .await
19402        .unwrap();
19403        git::git(&path, &["checkout", "--detach", &next])
19404            .await
19405            .unwrap();
19406        resumed.state.reviews.push(
19407            serde_json::from_value(serde_json::json!({
19408                "round":1, "head":next, "reviews":[]
19409            }))
19410            .unwrap(),
19411        );
19412        resumed.state.save().unwrap();
19413        let error = resumed.execute().await.unwrap_err();
19414        assert!(
19415            error
19416                .to_string()
19417                .contains("selected implementation is held"),
19418            "{error:#}"
19419        );
19420        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
19421    }
19422
19423    #[tokio::test]
19424    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
19425        crate::run::pin_test_home();
19426        let mut runner = runner_at(RunStatus::Gating);
19427        runner.state.save().unwrap();
19428        let id = runner.state.id.clone();
19429        let passes = runner.state.passes.len();
19430        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19431        let mut claiming = Box::pin(runner.claim_driver());
19432        std::future::poll_fn(|cx| {
19433            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
19434            std::task::Poll::Ready(())
19435        })
19436        .await;
19437        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
19438        drop(owner);
19439        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
19440            .await
19441            .unwrap()
19442            .unwrap();
19443        assert_eq!(execution.runner.state.passes.len(), passes + 1);
19444        assert!(execution.runner.state.resource_block.is_none());
19445        assert_eq!(
19446            execution.runner.state.liveness(false),
19447            crate::run::Liveness::Live
19448        );
19449        drop(execution);
19450        assert!(RunState::load(&id).unwrap().driver_exited);
19451    }
19452
19453    #[tokio::test]
19454    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
19455        crate::run::pin_test_home();
19456        let mut runner = runner_at(RunStatus::Gating);
19457        runner.state.save().unwrap();
19458        // run.json can commit, but the subsequent lease metadata write fails.
19459        // Cleanup must run after releasing the short ownership lock.
19460        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
19461        assert!(runner.claim_driver().await.is_err());
19462        assert!(runner.state.driver_exited);
19463        assert_eq!(runner.state.driver_pid, None);
19464        let disk = RunState::load(&runner.state.id).unwrap();
19465        assert!(disk.driver_exited);
19466        assert_eq!(disk.driver_pid, None);
19467        assert_eq!(disk.status, RunStatus::Gating);
19468    }
19469
19470    #[tokio::test]
19471    async fn an_early_graph_error_releases_its_driver() {
19472        crate::run::pin_test_home();
19473        let mut runner = runner_at(RunStatus::Blocked);
19474        runner.state.config.graph.selected_implementer = Some("selected".into());
19475        runner.state.save().unwrap();
19476        assert!(runner.execute().await.is_err());
19477        let disk = RunState::load(&runner.state.id).unwrap();
19478        assert!(disk.driver_exited);
19479        assert_eq!(disk.driver_pid, None);
19480        assert_eq!(disk.status, RunStatus::Blocked);
19481    }
19482
19483    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
19484        crate::run::MergeRepair {
19485            question: "repair-question".into(),
19486            head: "old-head".into(),
19487            pr: "https://github.com/o/r/pull/42".into(),
19488            reviews: vec![review_round(false, 1, 1, 1, false, true)],
19489            gate: vec![outcome(Some(0), false)],
19490            gate_head: Some("old-head".into()),
19491            gate_fixes: Vec::new(),
19492            pre_gate: Vec::new(),
19493            phase,
19494            failure: None,
19495        }
19496    }
19497
19498    #[tokio::test]
19499    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
19500        crate::run::pin_test_home();
19501        let mut runner = runner_at(RunStatus::Landing);
19502        runner.state.merge_repairs.push(merge_repair_request(
19503            crate::run::MergeRepairPhase::Requested,
19504        ));
19505        runner.state.save().unwrap();
19506        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
19507        assert!(
19508            runner
19509                .execute()
19510                .await
19511                .unwrap_err()
19512                .to_string()
19513                .contains("already running")
19514        );
19515        assert_eq!(
19516            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19517            crate::run::MergeRepairPhase::Requested
19518        );
19519        drop(claim);
19520        let mut disk = runner.state.clone();
19521        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
19522        disk.save().unwrap();
19523        assert!(
19524            runner
19525                .execute()
19526                .await
19527                .unwrap_err()
19528                .to_string()
19529                .contains("decision changed")
19530        );
19531        assert_eq!(
19532            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
19533            crate::run::MergeRepairPhase::Complete
19534        );
19535    }
19536
19537    #[test]
19538    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
19539        crate::run::pin_test_home();
19540        let mut runner = runner_at(RunStatus::Landing);
19541        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
19542        runner.state.reviews = request.reviews.clone();
19543        runner.state.gate = request.gate.clone();
19544        runner.state.gate_ran = true;
19545        runner.state.land_approval = Some(crate::run::LandApproval {
19546            question: "old".into(),
19547            head: "old-head".into(),
19548        });
19549        runner.state.merge_repairs.push(request);
19550        runner.begin_merge_repair_review(0).unwrap();
19551        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
19552        assert!(!runner.state.gate_status().ok());
19553        assert!(runner.state.land_approval.is_none());
19554        let disk = RunState::load(&runner.state.id).unwrap();
19555        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
19556        assert!(disk.merge_repairs[0].gate[0].ok());
19557        assert_eq!(
19558            disk.merge_repairs[0].phase,
19559            crate::run::MergeRepairPhase::Reviewing
19560        );
19561        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
19562        assert!(disk.reviews.is_empty());
19563    }
19564
19565    #[tokio::test]
19566    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
19567        crate::run::pin_test_home();
19568        for phase in [
19569            crate::run::MergeRepairPhase::Fixing,
19570            crate::run::MergeRepairPhase::Requested,
19571        ] {
19572            let mut runner = runner_at(RunStatus::Landing);
19573            runner.state.config.graph.land_rounds = 0;
19574            runner.state.merge_repairs.push(merge_repair_request(phase));
19575            runner.state.save().unwrap();
19576            assert!(!runner.repair_merge_review().await.unwrap());
19577            assert_eq!(runner.state.status, RunStatus::Blocked);
19578            assert_eq!(
19579                runner.state.merge_repairs[0].phase,
19580                crate::run::MergeRepairPhase::Failed
19581            );
19582            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
19583            assert!(
19584                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
19585                    "interrupted"
19586                } else {
19587                    "budget exhausted"
19588                })
19589            );
19590            assert!(runner.state.merge.is_none());
19591        }
19592    }
19593
19594    /// `park_here` folding in the reason `Pause::park_because` recorded -
19595    /// this is what lets an operator reading a run's events tell an
19596    /// interrupt-driven park from an ordinary shutdown park.
19597    #[test]
19598    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
19599        crate::run::pin_test_home();
19600        let mut runner = runner_at(RunStatus::Implementing);
19601        let interrupt = Pause::new();
19602        runner.watch_interrupt(interrupt.clone());
19603
19604        interrupt.park_because("task a1b2 asked to run first");
19605
19606        assert!(runner.park_here().expect("park_here"));
19607        assert!(runner.state.parked);
19608        let last = runner.state.events.last().expect("a park event");
19609        assert_eq!(last.node, "park");
19610        assert!(
19611            last.message.contains("task a1b2 asked to run first"),
19612            "expected the interrupt reason in {:?}",
19613            last.message
19614        );
19615    }
19616
19617    /// The cause is stored on the pass when it parks, and the first cause a
19618    /// shared pause was given is the one that stays.
19619    #[test]
19620    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
19621        crate::run::pin_test_home();
19622        let mut runner = runner_at(RunStatus::Implementing);
19623        runner.state.begin_pass();
19624        let shutdown = Pause::new();
19625        runner.on_pause(shutdown.clone());
19626        let upgrade = ParkCause::Upgrade {
19627            from: "0.121.0".to_owned(),
19628            to: Some("0.121.1".to_owned()),
19629        };
19630        shutdown.park_for(upgrade.clone());
19631        shutdown.park_for(ParkCause::Operator);
19632
19633        assert!(runner.park_here().expect("park_here"));
19634        assert_eq!(
19635            runner.state.passes[0].end,
19636            Some(crate::run::PassEnd::Parked { cause: upgrade })
19637        );
19638        let last = runner.state.events.last().expect("a park event");
19639        assert!(last.message.contains("0.121.0"), "{}", last.message);
19640    }
19641
19642    #[tokio::test]
19643    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
19644        crate::run::pin_test_home();
19645        let mut runner = runner_at(RunStatus::Reviewing);
19646        runner.state.save().unwrap();
19647        let mut execution = runner.claim_driver().await.expect("first execution");
19648        assert_eq!(execution.runner.state.passes.len(), 1);
19649        execution
19650            .driver
19651            .finish(&mut execution.runner.state)
19652            .await
19653            .unwrap();
19654    }
19655
19656    #[tokio::test]
19657    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
19658        crate::run::pin_test_home();
19659        let mut runner = runner_at(RunStatus::Reviewing);
19660        runner.state.config.agents = vec![conductor()];
19661        runner.state.config.graph.timeout_resource_wait = 0;
19662        runner.state.save().unwrap();
19663        let id = runner.state.id.clone();
19664        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
19665        assert!(runner.claim_driver().await.is_err());
19666        drop(owner);
19667        drop(runner);
19668        let mut resumed = Runner::resume(&id).unwrap();
19669        assert_eq!(resumed.state.driver_claimed, Some(false));
19670        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
19671        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
19672        let mut execution = resumed
19673            .claim_driver()
19674            .await
19675            .expect("retry initial claim after reload");
19676        assert_eq!(execution.runner.state.driver_claimed, Some(true));
19677        assert_eq!(execution.runner.state.passes.len(), 1);
19678        execution
19679            .driver
19680            .finish(&mut execution.runner.state)
19681            .await
19682            .unwrap();
19683    }
19684
19685    #[tokio::test]
19686    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
19687        crate::run::pin_test_home();
19688        let mut runner = runner_at(RunStatus::Reviewing);
19689        runner.state.config.agents = vec![conductor()];
19690        runner.state.save().unwrap();
19691        let id = runner.state.id.clone();
19692        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
19693        let mut resumed = Runner::resume(&id).unwrap();
19694        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
19695        assert!(resumed.claim_driver().await.is_err());
19696        assert!(RunState::load(&id).unwrap().passes.is_empty());
19697        drop(lease);
19698        let mut execution = resumed.claim_driver().await.unwrap();
19699        execution
19700            .driver
19701            .finish(&mut execution.runner.state)
19702            .await
19703            .unwrap();
19704    }
19705
19706    #[tokio::test]
19707    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
19708        crate::run::pin_test_home();
19709        let mut runner = runner_at(RunStatus::Reviewing);
19710        let state = &mut runner.state;
19711        state.config.agents = vec![conductor()];
19712        state.driver_claimed = None;
19713        state.driver_pid = Some(std::process::id());
19714        state.driver_started_at = None;
19715        state.begin_pass();
19716        state.save().unwrap();
19717        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
19718        std::fs::write(
19719            state.dir().join("driver.json"),
19720            serde_json::to_vec(&serde_json::json!({
19721                "pid": state.driver_pid, "started_at": null,
19722                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
19723            }))
19724            .unwrap(),
19725        )
19726        .unwrap();
19727        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
19728        drop(lease);
19729        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
19730        let mut resumed = Runner::resume(&state.id).unwrap();
19731        assert!(!resumed.state.driver_exited);
19732        let mut execution = resumed
19733            .claim_driver()
19734            .await
19735            .expect("lease proves previous execution ended");
19736        assert_eq!(execution.runner.state.passes.len(), 2);
19737        execution
19738            .driver
19739            .finish(&mut execution.runner.state)
19740            .await
19741            .unwrap();
19742    }
19743
19744    #[tokio::test]
19745    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
19746        crate::run::pin_test_home();
19747        let mut runner = runner_at(RunStatus::Reviewing);
19748        let mut json = serde_json::to_value(&runner.state).unwrap();
19749        json.as_object_mut().unwrap().remove("driver_claimed");
19750        runner.state = serde_json::from_value(json).unwrap();
19751        assert_eq!(runner.state.driver_claimed, None);
19752        runner.state.save().unwrap();
19753        let mut execution = runner
19754            .claim_driver()
19755            .await
19756            .expect("the execution lease, not a missing marker, excludes a second driver");
19757        assert_eq!(execution.runner.state.passes.len(), 1);
19758        execution
19759            .driver
19760            .finish(&mut execution.runner.state)
19761            .await
19762            .unwrap();
19763    }
19764
19765    #[tokio::test]
19766    async fn driver_admission_refuses_a_proven_live_driver() {
19767        crate::run::pin_test_home();
19768        let mut runner = runner_at(RunStatus::Reviewing);
19769        let pid = std::process::id();
19770        runner.state.driver_pid = Some(pid);
19771        runner.state.driver_started_at = crate::proc::process_started_at(pid);
19772        assert!(runner.state.driver_started_at.is_some());
19773        runner.state.save().unwrap();
19774        let error = runner
19775            .claim_driver()
19776            .await
19777            .err()
19778            .expect("a live driver must refuse");
19779        assert!(error.to_string().contains("live driver"), "{error:#}");
19780        let saved = RunState::load(&runner.state.id).unwrap();
19781        assert_eq!(saved.driver_pid, Some(pid));
19782        assert_eq!(saved.status, RunStatus::Reviewing);
19783        assert!(saved.passes.is_empty());
19784    }
19785
19786    #[test]
19787    fn parking_resource_halt_persists_the_pass_once() {
19788        crate::run::pin_test_home();
19789        let mut runner = runner_at(RunStatus::Reviewing);
19790        runner.state.begin_pass();
19791        runner.state.resource_block = Some(crate::run::ResourceBlock {
19792            node: "fix".into(),
19793            kind: "shared build cache".into(),
19794            detail: "cancelled".into(),
19795            at: Timestamp::now(),
19796            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
19797        });
19798        let cause = ParkCause::Upgrade {
19799            from: "old".into(),
19800            to: Some("new".into()),
19801        };
19802        runner.pause.park_for(cause.clone());
19803        // Both direct fixer exits and graph resource boundaries must complete
19804        // parking, rather than returning a stale Reviewing record.
19805        runner.stop_for_resource().unwrap();
19806        assert!(runner.resource_halt().unwrap());
19807        assert!(runner.park_here().unwrap());
19808        let saved = RunState::load(&runner.state.id).unwrap();
19809        assert!(saved.parked);
19810        assert_eq!(saved.status, RunStatus::Reviewing);
19811        assert!(saved.passes[0].ended_at.is_some());
19812        assert_eq!(
19813            saved.passes[0].end,
19814            Some(crate::run::PassEnd::Parked {
19815                cause: cause.clone()
19816            })
19817        );
19818        assert_eq!(
19819            saved.resource_block.unwrap().cause,
19820            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
19821        );
19822        assert_eq!(
19823            saved
19824                .events
19825                .iter()
19826                .filter(|event| event.node == "park")
19827                .count(),
19828            1
19829        );
19830    }
19831
19832    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
19833    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
19834    /// cleared) must not make a *different* run - one only watching its own,
19835    /// unshared interrupt `Pause` - see itself as parked. If a future change
19836    /// ever collapsed these back into one handle, the interrupt scheduler
19837    /// would park every run for the rest of the daemon's life, not just the
19838    /// one it meant to interrupt.
19839    #[test]
19840    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
19841        crate::run::pin_test_home();
19842        let mut runner = runner_at(RunStatus::Implementing);
19843        let shutdown = Pause::new();
19844        runner.on_pause(shutdown.clone());
19845        let interrupt = Pause::new();
19846        runner.watch_interrupt(interrupt.clone());
19847
19848        // Nobody has asked for anything yet.
19849        assert!(!runner.park_here().expect("park_here"));
19850        assert!(!runner.state.parked);
19851
19852        // Only the interrupt handle fires; the shutdown handle stays clear.
19853        interrupt.park_because("test");
19854        assert!(!shutdown.parked());
19855        assert!(runner.park_here().expect("park_here"));
19856    }
19857
19858    /// The property every prior attempt at this feature failed to pin down:
19859    /// asking a run to park while one of its nodes has a real, in-flight
19860    /// async operation running (an agent call, in production) must not cut
19861    /// that operation short. `park_here` is only ever consulted *between*
19862    /// `execute`'s node calls - see its own doc - so nothing inside a node
19863    /// can observe a park request until the node itself returns. This proves
19864    /// that structurally, with real `tokio` concurrency and a channel
19865    /// handshake (never a sleep, which would only prove "usually", not
19866    /// "cannot"): the "node" below reports that it has genuinely started,
19867    /// and only then is the park requested; the node still has to be told to
19868    /// finish before `park_here` is ever called, exactly mirroring every
19869    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
19870    /// in `execute`.
19871    #[tokio::test]
19872    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
19873        crate::run::pin_test_home();
19874        let mut runner = runner_at(RunStatus::Implementing);
19875        let interrupt = Pause::new();
19876        runner.watch_interrupt(interrupt.clone());
19877
19878        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
19879        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
19880
19881        // Stands in for one node's in-flight agent call: it proves it has
19882        // genuinely started, then blocks - exactly as a spawned CLI process
19883        // does - until told to finish.
19884        let node = async move {
19885            started_tx.send(()).expect("send started");
19886            finish_rx.await.expect("recv finish");
19887            "node finished"
19888        };
19889
19890        let interrupter = async move {
19891            started_rx.await.expect("recv started");
19892            // The call is now genuinely in flight. Ask it to park.
19893            interrupt.park_because("higher-priority task waiting");
19894            // Nothing the node does can observe this yet - there is no
19895            // check inside it, by construction - so let the executor run
19896            // anything pending and then let the node finish on its own.
19897            tokio::task::yield_now().await;
19898            finish_tx.send(()).expect("send finish");
19899        };
19900
19901        let (node_result, ()) = tokio::join!(node, interrupter);
19902        assert_eq!(
19903            node_result, "node finished",
19904            "the in-flight call ran to completion"
19905        );
19906
19907        // Only now, at the boundary the real `execute` would check right
19908        // after this node, does the park take effect.
19909        assert!(runner.park_here().expect("park_here"));
19910        assert!(runner.state.parked);
19911    }
19912
19913    /// A run parked mid-competition carries every field it had accumulated
19914    /// through the exact same disk round-trip an ordinary resume uses -
19915    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
19916    /// Nothing about parking for an interrupt is a special case of that path;
19917    /// this is what proves it rather than assuming it.
19918    #[test]
19919    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
19920        crate::run::pin_test_home();
19921        let mut runner = runner_at(RunStatus::Judging);
19922        // `Runner::resume` re-resolves roles from the saved config, which
19923        // refuses an empty roster - give it the same minimal one `conductor`
19924        // itself uses.
19925        runner.state.config.agents = vec![conductor()];
19926        runner.state.candidates = vec![Candidate {
19927            index: 0,
19928            label: 'A',
19929            agent: "alpha".to_owned(),
19930            branch: "magi/x/A".to_owned(),
19931            worktree: PathBuf::from("/nonexistent/worktree"),
19932            summary: "did the thing".to_owned(),
19933            stat: "1 file changed".to_owned(),
19934            files: 1,
19935            commits: 1,
19936            empty: false,
19937            failed: None,
19938            verified_noop: None,
19939            duration_ms: 1234,
19940            folded: false,
19941        }];
19942        let run_id = runner.state.id.clone();
19943
19944        let interrupt = Pause::new();
19945        runner.watch_interrupt(interrupt.clone());
19946        interrupt.park_because("task c3d4 asked to run first");
19947        assert!(runner.park_here().expect("park_here"));
19948
19949        let resumed = Runner::resume(&run_id).expect("resume");
19950        assert_eq!(resumed.state.candidates.len(), 1);
19951        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
19952        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
19953        assert_eq!(resumed.state.status, runner.state.status);
19954        assert!(
19955            resumed.state.parked,
19956            "still parked until `execute` actually walks the graph again"
19957        );
19958        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
19959    }
19960
19961    /// A fresh open question on `run`, stored and handed back for assertions.
19962    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
19963        let mut q = ask::Question::new(
19964            run.to_owned(),
19965            "implement".to_owned(),
19966            "impl-A".to_owned(),
19967            "Which storage backend should the cache use?".to_owned(),
19968            String::new(),
19969            vec!["SQLite".to_owned(), "Redis".to_owned()],
19970        );
19971        store.put(&mut q).unwrap();
19972        q
19973    }
19974
19975    #[test]
19976    fn a_failed_runs_open_question_is_abandoned() {
19977        ask_test_home();
19978        let store = ask::Questions::open();
19979        let mut runner = runner_at(RunStatus::Failed);
19980        let run = runner.state.id.clone();
19981        let q = ask_open_question(&store, &run);
19982
19983        runner.settle_questions();
19984
19985        let back = store.get(&q.id).unwrap();
19986        assert!(
19987            !back.status.open(),
19988            "the seat that asked died with the run; nobody is left to read an answer"
19989        );
19990        assert!(
19991            back.detail.contains(&run) && back.detail.contains("failed"),
19992            "the reason names what the run became, not just that it is gone: {}",
19993            back.detail
19994        );
19995    }
19996
19997    #[test]
19998    fn a_merged_runs_open_question_is_abandoned_too() {
19999        ask_test_home();
20000        let store = ask::Questions::open();
20001        // A run that finishes cleanly still leaves nobody to read an answer -
20002        // this is not only a failure-path cleanup.
20003        for status in [RunStatus::Merged, RunStatus::Ready] {
20004            let mut runner = runner_at(status);
20005            let run = runner.state.id.clone();
20006            let q = ask_open_question(&store, &run);
20007
20008            runner.settle_questions();
20009
20010            let back = store.get(&q.id).unwrap();
20011            assert!(
20012                !back.status.open(),
20013                "{status:?} run's question must not outlive the run"
20014            );
20015        }
20016    }
20017
20018    #[test]
20019    fn a_still_resumable_runs_open_question_is_left_alone() {
20020        ask_test_home();
20021        let store = ask::Questions::open();
20022        // `Blocked` and `Stalled` can still be resumed — the candidates, the
20023        // review round and the seat sessions are all still on disk — so a
20024        // question asked mid-round may yet get a real answer from a real
20025        // resume. Sweeping it here would be exactly the failure mode this
20026        // whole feature exists to avoid on the other side.
20027        for status in [RunStatus::Blocked, RunStatus::Stalled] {
20028            let mut runner = runner_at(status);
20029            let run = runner.state.id.clone();
20030            let q = ask_open_question(&store, &run);
20031
20032            runner.settle_questions();
20033
20034            let back = store.get(&q.id).unwrap();
20035            assert!(
20036                back.status.open(),
20037                "{status:?} is still alive; the question must still be waiting"
20038            );
20039        }
20040    }
20041
20042    #[test]
20043    fn settle_questions_never_touches_an_already_answered_question() {
20044        ask_test_home();
20045        let store = ask::Questions::open();
20046        let mut runner = runner_at(RunStatus::Failed);
20047        let run = runner.state.id.clone();
20048        let mut q = ask_open_question(&store, &run);
20049        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
20050            .unwrap();
20051        store.put(&mut q).unwrap();
20052
20053        // Called twice, the way a crash-recovered daemon reclaim and the
20054        // graph's own cleanup both can for the same run — `abandon_for_run`
20055        // only ever touches what is still open, so this must be inert both
20056        // times, not merely the second.
20057        runner.settle_questions();
20058        runner.settle_questions();
20059
20060        let back = store.get(&q.id).unwrap();
20061        assert_eq!(
20062            back.status,
20063            ask::QuestionStatus::Answered,
20064            "a real answer is a decision on record, never overwritten by a sweep"
20065        );
20066    }
20067
20068    #[tokio::test]
20069    async fn fold_reclaims_only_proven_idle_isolated_outputs() {
20070        let tmp = tempfile::tempdir().unwrap();
20071        let home = tmp.path().join("home");
20072        let repo = tmp.path().join("repo");
20073        std::fs::create_dir(&home).unwrap();
20074        std::fs::create_dir(&repo).unwrap();
20075        init_repo(&repo);
20076
20077        let mut config = Config::default();
20078        config.graph.worktree_root = Some(tmp.path().join("wt"));
20079        config.verify.build_cache = crate::config::BuildCache::Isolated;
20080        let mut state = RunState::new(
20081            repo.clone(),
20082            "main".into(),
20083            "base".into(),
20084            "fold fixture".into(),
20085            config,
20086        );
20087        let root = state.worktree_root();
20088        let mut outputs = Vec::new();
20089        for name in ["idle", "busy"] {
20090            let checkout = root.join(name);
20091            git::worktree_add_detached(&repo, &checkout, "main")
20092                .await
20093                .unwrap();
20094            let target = crate::build_target::resolve(&home, &checkout).unwrap();
20095            let output = target.join("generated");
20096            std::fs::write(&output, name).unwrap();
20097            outputs.push((target, output));
20098        }
20099        let unknown = crate::build_target::root(&home)
20100            .join("00000000-0000-0000-0000-000000000000")
20101            .join("target");
20102        std::fs::create_dir_all(&unknown).unwrap();
20103        let unowned = unknown.join("generated");
20104        std::fs::write(&unowned, "keep").unwrap();
20105        let owner = crate::cache::Owner::here("fixture", "build", "busy", &root.join("busy"), "");
20106        let crate::cache::AcquireOutcome::Acquired(_lease) =
20107            crate::cache::try_acquire(&home, &outputs[1].0, &owner).unwrap()
20108        else {
20109            panic!("fixture target must be free");
20110        };
20111
20112        fold_run(&mut state, true, &home).await.unwrap();
20113        assert!(!root.exists());
20114        assert!(!outputs[0].1.exists());
20115        assert!(outputs[1].1.is_file());
20116        assert!(unowned.is_file());
20117    }
20118
20119    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
20120    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
20121    /// without merging, whose winner is still the operator's answer to read.
20122    /// Nothing previously called `fold_run` itself with a real `tally`, so
20123    /// this is the first test to pin down the one distinction the whole
20124    /// automatic-fold feature depends on: the winner's worktree and branch
20125    /// must survive, everything else sharing the run's worktree bay - a
20126    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
20127    /// second sweep treats every non-winner directory under the bay alike -
20128    /// must not.
20129    #[tokio::test]
20130    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
20131        crate::run::pin_test_home();
20132        let tmp = tempfile::tempdir().expect("tempdir");
20133        let repo = tmp.path().join("repo");
20134        std::fs::create_dir_all(&repo).unwrap();
20135        init_repo(&repo);
20136
20137        let mut config = Config::default();
20138        config.graph.worktree_root = Some(tmp.path().join("wt"));
20139
20140        let mut state = RunState::new(
20141            repo.clone(),
20142            "main".to_owned(),
20143            "deadbeef".to_owned(),
20144            "task".to_owned(),
20145            config,
20146        );
20147        let root = state.worktree_root();
20148        let wt_a = root.join("cand-A");
20149        let wt_b = root.join("cand-B");
20150        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
20151            .await
20152            .expect("worktree A");
20153        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
20154            .await
20155            .expect("worktree B");
20156
20157        state.candidates = vec![
20158            Candidate {
20159                index: 0,
20160                label: 'A',
20161                agent: "alpha".to_owned(),
20162                branch: "magi/x/A".to_owned(),
20163                worktree: wt_a.clone(),
20164                summary: String::new(),
20165                stat: String::new(),
20166                files: 0,
20167                commits: 0,
20168                empty: false,
20169                failed: None,
20170                verified_noop: None,
20171                duration_ms: 0,
20172                folded: false,
20173            },
20174            Candidate {
20175                index: 1,
20176                label: 'B',
20177                agent: "beta".to_owned(),
20178                branch: "magi/x/B".to_owned(),
20179                worktree: wt_b.clone(),
20180                summary: String::new(),
20181                stat: String::new(),
20182                files: 0,
20183                commits: 0,
20184                empty: false,
20185                failed: None,
20186                verified_noop: None,
20187                duration_ms: 0,
20188                folded: false,
20189            },
20190        ];
20191        state.tally = Some(Tally {
20192            first_choice: BTreeMap::from([('A', 1)]),
20193            borda: BTreeMap::new(),
20194            winner: 'A',
20195            rankings: 1,
20196            unanimous_initial: true,
20197            deliberated: false,
20198            changed_votes: 0,
20199            unanimous_final: true,
20200            tie_break: None,
20201            judges: 1,
20202            present: 1,
20203            quorum: 1,
20204            met_quorum: true,
20205            uncontested: None,
20206        });
20207        state.status = RunStatus::Ready;
20208
20209        fold_run(&mut state, false, &crate::run::home())
20210            .await
20211            .expect("fold_run");
20212
20213        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
20214        assert!(
20215            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
20216            "the unmerged winner's branch survives"
20217        );
20218        assert!(
20219            !state.candidates[0].folded,
20220            "the winner is not marked folded"
20221        );
20222
20223        assert!(!wt_b.exists(), "the loser's worktree is removed");
20224        assert!(
20225            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
20226            "the loser's branch is removed"
20227        );
20228        assert!(state.candidates[1].folded, "the loser is marked folded");
20229    }
20230
20231    #[tokio::test]
20232    #[cfg(unix)]
20233    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
20234        use std::os::fd::AsRawFd as _;
20235        use std::time::Duration;
20236
20237        let tmp = tempfile::tempdir().unwrap();
20238        let repo = tmp.path().join("repo");
20239        std::fs::create_dir_all(&repo).unwrap();
20240        init_repo(&repo);
20241        let common = repo.join(".git");
20242        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
20243        let home = tmp.path().join("home");
20244        let mut config = Config::default();
20245        config.coordination = crate::config::test_coordination();
20246        config.graph.worktree_root = Some(tmp.path().join("wt"));
20247        config.graph.timeout_resource_wait = 1;
20248        let mut state = RunState::new(
20249            repo.clone(),
20250            "main".into(),
20251            "deadbeef".into(),
20252            "task".into(),
20253            config,
20254        );
20255        let root = state.worktree_root();
20256        let candidate = root.join("cand-A");
20257        let reviewer = root.join("review-1");
20258        let branch = state.branch_for('A');
20259        git::worktree_add_branch(&repo, &candidate, &branch, "main")
20260            .await
20261            .unwrap();
20262        git::worktree_add_detached(&repo, &reviewer, "main")
20263            .await
20264            .unwrap();
20265        git::acquire_worktree_config(&repo).await.unwrap();
20266        state.enabled_worktree_config = true;
20267        state.status = RunStatus::Ready;
20268        state.candidates.push(Candidate {
20269            index: 0,
20270            label: 'A',
20271            agent: "implement".into(),
20272            branch: branch.clone(),
20273            worktree: candidate.clone(),
20274            summary: String::new(),
20275            stat: String::new(),
20276            files: 0,
20277            commits: 0,
20278            empty: false,
20279            failed: None,
20280            verified_noop: None,
20281            duration_ms: 0,
20282            folded: false,
20283        });
20284        state.save_under(&home).unwrap();
20285        let config_before = std::fs::read(common.join("config")).unwrap();
20286        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
20287            .await
20288            .unwrap();
20289        // A separate OS handle bypasses the launcher's local gate, exactly as
20290        // an external updater does, and must still prevent all folding work.
20291        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
20292        assert_eq!(
20293            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
20294            0
20295        );
20296        let mut fold = Box::pin(fold_run(&mut state, true, &home));
20297        assert!(
20298            tokio::time::timeout(Duration::from_millis(200), &mut fold)
20299                .await
20300                .is_err()
20301        );
20302        assert!(candidate.exists());
20303        assert!(reviewer.exists());
20304        assert!(git::branch_exists(&repo, &branch).await.unwrap());
20305        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20306        assert_eq!(
20307            git::git(&repo, &["worktree", "list", "--porcelain"])
20308                .await
20309                .unwrap(),
20310            worktrees_before
20311        );
20312        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
20313            .await
20314            .unwrap()
20315            .unwrap_err();
20316        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
20317        drop(fold);
20318        assert!(candidate.exists() && reviewer.exists());
20319        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
20320        assert_eq!(
20321            state.resource_block.as_ref().unwrap().kind,
20322            "repository coordination"
20323        );
20324        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
20325        drop(external);
20326        let mut fold = Box::pin(fold_run(&mut state, true, &home));
20327        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
20328            .await
20329            .unwrap()
20330            .unwrap();
20331        drop(fold);
20332        assert!(removed.contains(&branch));
20333        assert!(!root.exists());
20334        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
20335        assert!(state.candidates[0].folded);
20336        assert!(!state.enabled_worktree_config);
20337        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
20338        let enabled = git::git_raw(
20339            &repo,
20340            &["config", "--local", "--get", "extensions.worktreeConfig"],
20341        )
20342        .await
20343        .unwrap();
20344        assert!(!enabled.ok(), "the last config reference was released");
20345    }
20346
20347    /// A branch handed to a later run is that run's (and its pull request's):
20348    /// folding the run that released it must not delete it.
20349    #[tokio::test]
20350    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
20351        let tmp = tempfile::tempdir().expect("tempdir");
20352        let repo = tmp.path().join("repo");
20353        std::fs::create_dir_all(&repo).unwrap();
20354        init_repo(&repo);
20355        let home = tmp.path().join("home");
20356
20357        let mut config = Config::default();
20358        config.graph.worktree_root = Some(tmp.path().join("wt"));
20359        let mut state = RunState::new(
20360            repo.clone(),
20361            "main".to_owned(),
20362            "deadbeef".to_owned(),
20363            "task".to_owned(),
20364            config,
20365        );
20366        // The worktree is already gone (released); the branch survives.
20367        git::git(&repo, &["branch", "magi/x/A", "main"])
20368            .await
20369            .expect("branch");
20370        state.candidates = vec![Candidate {
20371            index: 0,
20372            label: 'A',
20373            agent: "alpha".to_owned(),
20374            branch: "magi/x/A".to_owned(),
20375            worktree: state.worktree_root().join("cand-A"),
20376            summary: String::new(),
20377            stat: String::new(),
20378            files: 0,
20379            commits: 0,
20380            empty: false,
20381            failed: None,
20382            verified_noop: None,
20383            duration_ms: 0,
20384            folded: true,
20385        }];
20386        state.released_to = Some("20260901-000000-new1".to_owned());
20387        state.released_branches = vec!["magi/x/A".to_owned()];
20388
20389        fold_run(&mut state, true, &home).await.expect("fold_run");
20390        state.status = RunStatus::Merged;
20391        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
20392        state.merge = Some(crate::run::MergeOutcome {
20393            mode: MergeMode::Pr,
20394            ok: true,
20395            detail: "confirmed merged".into(),
20396            empty: false,
20397        });
20398        fold_run(&mut state, true, &home)
20399            .await
20400            .expect("merged fold");
20401
20402        assert!(
20403            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
20404            "the handed-over branch survives a fold"
20405        );
20406    }
20407
20408    /// A winner with nothing ahead of the base is caught before `gh` is ever
20409    /// asked for a pull request, and the message carries what the task's
20410    /// references resolved to.
20411    #[tokio::test]
20412    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
20413        let tmp = tempfile::tempdir().expect("tempdir");
20414        let repo = tmp.path().join("repo");
20415        std::fs::create_dir_all(&repo).unwrap();
20416        init_repo(&repo);
20417        let run = |args: &[&str]| {
20418            let out = std::process::Command::new("git")
20419                .quiet()
20420                .args(args)
20421                .current_dir(&repo)
20422                .output()
20423                .expect("spawn git");
20424            assert!(out.status.success(), "git {args:?}");
20425        };
20426        run(&["branch", "magi/x/A"]);
20427        run(&["checkout", "-q", "-b", "magi/x/B"]);
20428        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
20429        run(&["add", "-A"]);
20430        run(&["commit", "-q", "-m", "work"]);
20431        run(&["checkout", "-q", "main"]);
20432
20433        // A pull request is compared against the remote's base, so the
20434        // fixture needs one. Before it exists nothing can be read, and the
20435        // answer must be "not empty".
20436        let probe = RunState::new(
20437            repo.clone(),
20438            "main".to_owned(),
20439            "deadbeef".to_owned(),
20440            "task".to_owned(),
20441            Config::default(),
20442        );
20443        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
20444        let bare = tmp.path().join("origin.git");
20445        let out = std::process::Command::new("git")
20446            .quiet()
20447            .args(["init", "-q", "--bare"])
20448            .arg(&bare)
20449            .output()
20450            .expect("spawn git");
20451        assert!(out.status.success(), "git init --bare");
20452        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
20453        run(&["push", "-q", "origin", "main"]);
20454
20455        let mut state = RunState::new(
20456            repo.clone(),
20457            "main".to_owned(),
20458            "deadbeef".to_owned(),
20459            "task".to_owned(),
20460            Config::default(),
20461        );
20462        state.seeds = vec![refs::Seed {
20463            token: "magi/27b2/A".to_owned(),
20464            kind: refs::SeedKind::Unresolved,
20465            sha: String::new(),
20466            branch: true,
20467            detail: "no branch or commit named magi/27b2/A".to_owned(),
20468        }];
20469
20470        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
20471        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
20472        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
20473        let detail = empty_candidate_detail(&state, "main");
20474        assert!(detail.starts_with("empty candidate"), "{detail}");
20475        assert!(!detail.contains("magi/27b2/A"), "{detail}");
20476        assert!(detail.contains("--start-from"), "{detail}");
20477    }
20478
20479    /// `status == Ready` used to be read as "this is the harmless
20480    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
20481    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
20482    /// run's PR was closed without merging — and reentering `merge` with
20483    /// `mode` still `Pr` does not know the difference, so it pushed and
20484    /// opened a second pull request. `mode == Local` reproduces the same
20485    /// blind spot without a network call: reentry must not attempt another
20486    /// git merge once this node has already recorded an outcome.
20487    #[tokio::test]
20488    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
20489        let tmp = tempfile::tempdir().expect("tempdir");
20490        let repo = tmp.path().join("repo");
20491        std::fs::create_dir_all(&repo).unwrap();
20492        init_repo(&repo);
20493
20494        let mut config = Config::default();
20495        config.merge.mode = MergeMode::Local;
20496
20497        let mut state = RunState::new(
20498            repo.clone(),
20499            "main".to_owned(),
20500            "deadbeef".to_owned(),
20501            "task".to_owned(),
20502            config,
20503        );
20504        state.candidates = vec![Candidate {
20505            index: 0,
20506            label: 'A',
20507            agent: "alpha".to_owned(),
20508            branch: "does-not-exist".to_owned(),
20509            worktree: repo.clone(),
20510            summary: String::new(),
20511            stat: String::new(),
20512            files: 0,
20513            commits: 0,
20514            empty: false,
20515            failed: None,
20516            verified_noop: None,
20517            duration_ms: 0,
20518            folded: false,
20519        }];
20520        state.tally = Some(Tally {
20521            first_choice: BTreeMap::from([('A', 1)]),
20522            borda: BTreeMap::new(),
20523            winner: 'A',
20524            rankings: 1,
20525            unanimous_initial: true,
20526            deliberated: false,
20527            changed_votes: 0,
20528            unanimous_final: true,
20529            tie_break: None,
20530            judges: 0,
20531            present: 0,
20532            quorum: 0,
20533            met_quorum: true,
20534            uncontested: Some("only candidate A produced a change".to_owned()),
20535        });
20536        state.reviews = vec![ReviewRound {
20537            round: 1,
20538            head: "deadbeef".to_owned(),
20539            verified_head: None,
20540            verified_at: None,
20541            reviews: Vec::new(),
20542            e2e: Vec::new(),
20543            fix: None,
20544            blocking: 0,
20545            answered: 0,
20546            expected: 0,
20547            clean: true,
20548            verify_retried: false,
20549            e2e_deferred: false,
20550            e2e_defer_reason: None,
20551            progressed: false,
20552            vote_split: false,
20553            reconsideration: Vec::new(),
20554            verdict: None,
20555        }];
20556        state.gate = vec![CommandOutcome {
20557            command: "test".to_owned(),
20558            code: Some(0),
20559            output_tail: String::new(),
20560            duration_ms: 0,
20561            resource_blocked: false,
20562        }];
20563        state.gate_ran = true;
20564        // Reached its conclusion already — e.g. `land` closing the PR without
20565        // merging it, which (like the honest `MergeMode::None` path) leaves
20566        // `status` at `Ready`. The recorded outcome is what actually marks
20567        // this node done.
20568        state.status = RunStatus::Ready;
20569        state.merge = Some(MergeOutcome {
20570            mode: MergeMode::Local,
20571            ok: false,
20572            detail: "already concluded".to_owned(),
20573            empty: false,
20574        });
20575
20576        let mut runner = Runner {
20577            state,
20578            roles: ResolvedRoles {
20579                implementers: Vec::new(),
20580                judges: Vec::new(),
20581                reviewers: Vec::new(),
20582                fixer: None,
20583                conductor: conductor(),
20584                implementer_roster: Vec::new(),
20585                judge_roster: Vec::new(),
20586                reviewer_roster: Vec::new(),
20587            },
20588            sem: Arc::new(Semaphore::new(1)),
20589            pause: Pause::new(),
20590            interrupt: Pause::new(),
20591        };
20592
20593        runner.merge().await.expect("merge");
20594
20595        assert_eq!(
20596            runner.state.status,
20597            RunStatus::Ready,
20598            "a concluded run's status must not change on reentry"
20599        );
20600        assert_eq!(
20601            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20602            Some("already concluded"),
20603            "merge must not run again once the node already recorded an outcome"
20604        );
20605    }
20606
20607    /// `gate` leaves `state.gate_ran` false both before it has ever run and
20608    /// when its last attempt was resource-blocked (the shared build cache
20609    /// could not be acquired or confirmed fresh in time - see
20610    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
20611    /// `Vec` this also leaves behind used to read as "nothing failed" and let
20612    /// a run merge a tree the gate never actually checked - exactly the case
20613    /// a contended cache produces on every retry until it clears. `merge`
20614    /// must refuse until `gate` has actually recorded an attempt.
20615    #[tokio::test]
20616    async fn merge_refuses_a_gate_that_has_not_actually_run() {
20617        let tmp = tempfile::tempdir().expect("tempdir");
20618        let repo = tmp.path().join("repo");
20619        std::fs::create_dir_all(&repo).unwrap();
20620        init_repo(&repo);
20621
20622        let mut config = Config::default();
20623        config.merge.mode = MergeMode::Local;
20624
20625        let mut state = RunState::new(
20626            repo.clone(),
20627            "main".to_owned(),
20628            "deadbeef".to_owned(),
20629            "task".to_owned(),
20630            config,
20631        );
20632        state.candidates = vec![Candidate {
20633            index: 0,
20634            label: 'A',
20635            agent: "alpha".to_owned(),
20636            branch: "does-not-exist".to_owned(),
20637            worktree: repo.clone(),
20638            summary: String::new(),
20639            stat: String::new(),
20640            files: 0,
20641            commits: 0,
20642            empty: false,
20643            failed: None,
20644            verified_noop: None,
20645            duration_ms: 0,
20646            folded: false,
20647        }];
20648        state.tally = Some(Tally {
20649            first_choice: BTreeMap::from([('A', 1)]),
20650            borda: BTreeMap::new(),
20651            winner: 'A',
20652            rankings: 1,
20653            unanimous_initial: true,
20654            deliberated: false,
20655            changed_votes: 0,
20656            unanimous_final: true,
20657            tie_break: None,
20658            judges: 0,
20659            present: 0,
20660            quorum: 0,
20661            met_quorum: true,
20662            uncontested: Some("only candidate A produced a change".to_owned()),
20663        });
20664        state.reviews = vec![ReviewRound {
20665            round: 1,
20666            head: "deadbeef".to_owned(),
20667            verified_head: None,
20668            verified_at: None,
20669            reviews: Vec::new(),
20670            e2e: Vec::new(),
20671            fix: None,
20672            blocking: 0,
20673            answered: 0,
20674            expected: 0,
20675            clean: true,
20676            verify_retried: false,
20677            e2e_deferred: false,
20678            e2e_defer_reason: None,
20679            progressed: false,
20680            vote_split: false,
20681            reconsideration: Vec::new(),
20682            verdict: None,
20683        }];
20684        // The point: `gate` has not recorded anything yet.
20685        state.gate = Vec::new();
20686        state.gate_ran = false;
20687        state.status = RunStatus::Gating;
20688
20689        let mut runner = Runner {
20690            state,
20691            roles: ResolvedRoles {
20692                implementers: Vec::new(),
20693                judges: Vec::new(),
20694                reviewers: Vec::new(),
20695                fixer: None,
20696                conductor: conductor(),
20697                implementer_roster: Vec::new(),
20698                judge_roster: Vec::new(),
20699                reviewer_roster: Vec::new(),
20700            },
20701            sem: Arc::new(Semaphore::new(1)),
20702            pause: Pause::new(),
20703            interrupt: Pause::new(),
20704        };
20705
20706        runner.merge().await.expect("merge");
20707
20708        assert!(
20709            runner.state.merge.is_none(),
20710            "an empty gate must never be read as a passing one: {:?}",
20711            runner.state.merge
20712        );
20713    }
20714
20715    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
20716    /// commands configured and `merge.mode` is `none` (a review-only run).
20717    /// `gate` must still record a real attempt — zero commands, vacuously
20718    /// passed — rather than leaving `state.gate` empty in a way `merge`
20719    /// cannot tell apart from "never ran"; otherwise the run reaches
20720    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
20721    #[tokio::test]
20722    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
20723        let tmp = tempfile::tempdir().expect("tempdir");
20724        let repo = tmp.path().join("repo");
20725        std::fs::create_dir_all(&repo).unwrap();
20726        init_repo(&repo);
20727
20728        // Default config: `verify.gate` empty, `merge.mode` is `none`.
20729        let config = Config::default();
20730
20731        let mut state = RunState::new(
20732            repo.clone(),
20733            "main".to_owned(),
20734            "deadbeef".to_owned(),
20735            "task".to_owned(),
20736            config,
20737        );
20738        state.candidates = vec![Candidate {
20739            index: 0,
20740            label: 'A',
20741            agent: "alpha".to_owned(),
20742            branch: "does-not-exist".to_owned(),
20743            worktree: repo.clone(),
20744            summary: String::new(),
20745            stat: String::new(),
20746            files: 0,
20747            commits: 0,
20748            empty: false,
20749            failed: None,
20750            verified_noop: None,
20751            duration_ms: 0,
20752            folded: false,
20753        }];
20754        state.tally = Some(Tally {
20755            first_choice: BTreeMap::from([('A', 1)]),
20756            borda: BTreeMap::new(),
20757            winner: 'A',
20758            rankings: 1,
20759            unanimous_initial: true,
20760            deliberated: false,
20761            changed_votes: 0,
20762            unanimous_final: true,
20763            tie_break: None,
20764            judges: 0,
20765            present: 0,
20766            quorum: 0,
20767            met_quorum: true,
20768            uncontested: Some("only candidate A produced a change".to_owned()),
20769        });
20770        state.reviews = vec![ReviewRound {
20771            round: 1,
20772            head: "deadbeef".to_owned(),
20773            verified_head: None,
20774            verified_at: None,
20775            reviews: Vec::new(),
20776            e2e: Vec::new(),
20777            fix: None,
20778            blocking: 0,
20779            answered: 0,
20780            expected: 0,
20781            clean: true,
20782            verify_retried: false,
20783            e2e_deferred: false,
20784            e2e_defer_reason: None,
20785            progressed: false,
20786            vote_split: false,
20787            reconsideration: Vec::new(),
20788            verdict: None,
20789        }];
20790
20791        let mut runner = Runner {
20792            state,
20793            roles: ResolvedRoles {
20794                implementers: Vec::new(),
20795                judges: Vec::new(),
20796                reviewers: Vec::new(),
20797                fixer: None,
20798                conductor: conductor(),
20799                implementer_roster: Vec::new(),
20800                judge_roster: Vec::new(),
20801                reviewer_roster: Vec::new(),
20802            },
20803            sem: Arc::new(Semaphore::new(1)),
20804            pause: Pause::new(),
20805            interrupt: Pause::new(),
20806        };
20807
20808        runner.gate().await.expect("gate");
20809        assert!(
20810            runner.state.gate_ran,
20811            "zero configured commands is still a real attempt, not an unrun gate"
20812        );
20813        assert!(runner.state.gate.is_empty());
20814        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
20815        assert_ne!(
20816            runner.state.status,
20817            RunStatus::Blocked,
20818            "a gate with nothing to check must not read as failed"
20819        );
20820
20821        runner.merge().await.expect("merge");
20822        assert_eq!(
20823            runner.state.status,
20824            RunStatus::Ready,
20825            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
20826        );
20827    }
20828
20829    /// `Config::cache_dir` is derived from `verify.e2e` as well as
20830    /// `verify.gate` (so the e2e leg and the final gate never build against
20831    /// different directories). With zero `verify.gate` commands but a
20832    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
20833    /// that lease before discovering it had nothing to run - so a repo with
20834    /// no gate commands could come back `resource_blocked` (and therefore
20835    /// still `gate_ran == false`) on nothing but an unrelated run holding the
20836    /// cache, exactly the contention this run's own zero commands could
20837    /// never have touched. `gate` must recognise there is nothing to check
20838    /// before it ever asks for the lease.
20839    #[tokio::test]
20840    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
20841        crate::run::pin_test_home();
20842        let home = crate::run::home();
20843
20844        let tmp = tempfile::tempdir().expect("tempdir");
20845        let repo = tmp.path().join("repo");
20846        std::fs::create_dir_all(&repo).unwrap();
20847        init_repo(&repo);
20848        // Unique to this test, so holding its lease cannot collide with
20849        // another test sharing the same process-wide `home`.
20850        let cache_dir = tmp.path().join("target");
20851
20852        let mut config = Config::default();
20853        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
20854        // `verify.gate` stays empty (the default). Any accidental cache
20855        // acquisition must report contention immediately, regardless of how
20856        // long unrelated Git subprocesses take under parallel test load.
20857        config.graph.timeout_verify = Some(2);
20858        config.graph.timeout_resource_wait = 0;
20859
20860        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20861        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20862            .expect("no io error acquiring directly")
20863        {
20864            crate::cache::AcquireOutcome::Acquired(g) => g,
20865            crate::cache::AcquireOutcome::Busy(b) => {
20866                panic!("expected the direct acquire to win the lease first: {b:?}")
20867            }
20868        };
20869
20870        let mut state = RunState::new(
20871            repo.clone(),
20872            "main".to_owned(),
20873            "deadbeef".to_owned(),
20874            "task".to_owned(),
20875            config,
20876        );
20877        state.candidates = vec![Candidate {
20878            index: 0,
20879            label: 'A',
20880            agent: "alpha".to_owned(),
20881            branch: "does-not-exist".to_owned(),
20882            worktree: repo.clone(),
20883            summary: String::new(),
20884            stat: String::new(),
20885            files: 0,
20886            commits: 0,
20887            empty: false,
20888            failed: None,
20889            verified_noop: None,
20890            duration_ms: 0,
20891            folded: false,
20892        }];
20893        state.tally = Some(Tally {
20894            first_choice: BTreeMap::from([('A', 1)]),
20895            borda: BTreeMap::new(),
20896            winner: 'A',
20897            rankings: 1,
20898            unanimous_initial: true,
20899            deliberated: false,
20900            changed_votes: 0,
20901            unanimous_final: true,
20902            tie_break: None,
20903            judges: 0,
20904            present: 0,
20905            quorum: 0,
20906            met_quorum: true,
20907            uncontested: Some("only candidate A produced a change".to_owned()),
20908        });
20909        state.reviews = vec![ReviewRound {
20910            round: 1,
20911            head: "deadbeef".to_owned(),
20912            verified_head: None,
20913            verified_at: None,
20914            reviews: Vec::new(),
20915            e2e: Vec::new(),
20916            fix: None,
20917            blocking: 0,
20918            answered: 0,
20919            expected: 0,
20920            clean: true,
20921            verify_retried: false,
20922            e2e_deferred: false,
20923            e2e_defer_reason: None,
20924            progressed: false,
20925            vote_split: false,
20926            reconsideration: Vec::new(),
20927            verdict: None,
20928        }];
20929
20930        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
20931        state.reviews[0].e2e = vec![CommandOutcome {
20932            command: state.config.verify.e2e[0].clone(),
20933            code: Some(0),
20934            output_tail: String::new(),
20935            duration_ms: 0,
20936            resource_blocked: false,
20937        }];
20938        let mut runner = Runner {
20939            state,
20940            roles: ResolvedRoles {
20941                implementers: Vec::new(),
20942                judges: Vec::new(),
20943                reviewers: Vec::new(),
20944                fixer: None,
20945                conductor: conductor(),
20946                implementer_roster: Vec::new(),
20947                judge_roster: Vec::new(),
20948                reviewer_roster: Vec::new(),
20949            },
20950            sem: Arc::new(Semaphore::new(1)),
20951            pause: Pause::new(),
20952            interrupt: Pause::new(),
20953        };
20954
20955        runner.gate().await.expect("gate");
20956        assert!(runner.state.resource_block.is_none());
20957        assert!(
20958            !runner
20959                .state
20960                .events
20961                .iter()
20962                .any(|event| { event.message.contains("waiting for the shared build cache") }),
20963            "a gate with nothing to run must never request the held cache lease"
20964        );
20965        assert!(
20966            runner.state.gate_ran,
20967            "zero commands is still a real, immediate attempt"
20968        );
20969        assert!(runner.state.gate.is_empty());
20970        assert_ne!(
20971            runner.state.status,
20972            RunStatus::Blocked,
20973            "must not read as resource-blocked on a lease it never asked for"
20974        );
20975    }
20976
20977    /// A separate process carries a live worker's environment throughout all
20978    /// verification entry points; the parallel harness never changes its env.
20979    #[cfg(unix)]
20980    #[tokio::test]
20981    async fn verification_is_standalone_inside_a_live_worker() {
20982        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
20983        if std::env::var_os(CHILD).is_none() {
20984            let home = tempfile::tempdir().unwrap();
20985            let generation = crate::worker::pin_current(home.path()).unwrap();
20986            let output = std::process::Command::new(std::env::current_exe().unwrap())
20987                .args([
20988                    "--exact",
20989                    "graph::tests::verification_is_standalone_inside_a_live_worker",
20990                    "--nocapture",
20991                ])
20992                .env(CHILD, "1")
20993                .env("MAGI_HOME", home.path())
20994                .env("MAGI_WORKER_HOME", home.path())
20995                .env("MAGI_WORKER_ID", "abcdef")
20996                .env("MAGI_WORKER_GENERATION", &generation.id)
20997                .env("MAGI_WORKER_EXE", &generation.executable)
20998                .env("MAGI_RUN", "verification-parent")
20999                .env("MAGI_NODE", "gate")
21000                .env("MAGI_SEAT", "worker-fixture")
21001                .env("MAGI_PROCESS_OWNER", "fixture-owner")
21002                .env("MAGI_NO_AUTOUPDATE", "0")
21003                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
21004                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
21005                .env(
21006                    "VERIFY_EXPECT_TOOLCHAIN",
21007                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
21008                )
21009                .env(
21010                    "VERIFY_EXPECT_CACHE",
21011                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
21012                )
21013                .output()
21014                .unwrap();
21015            assert!(
21016                output.status.success()
21017                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
21018                "{}\n{}",
21019                String::from_utf8_lossy(&output.stdout),
21020                String::from_utf8_lossy(&output.stderr)
21021            );
21022            return;
21023        }
21024        let home = crate::run::home();
21025        let directory = home.join("workers/jobs/abcdef");
21026        std::fs::create_dir_all(&directory).unwrap();
21027        let job = crate::worker::Job {
21028            id: "abcdef".into(),
21029            sequence: 1,
21030            kind: "task".into(),
21031            key: "task:verification".into(),
21032            payload: serde_json::json!({}),
21033            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
21034            version: Some(env!("CARGO_PKG_VERSION").into()),
21035            status: crate::worker::Status::Running,
21036            result: None,
21037            pid: Some(std::process::id()),
21038            created_at: 0,
21039            finished_at: None,
21040        };
21041        std::fs::write(
21042            directory.join("job.json"),
21043            serde_json::to_vec(&job).unwrap(),
21044        )
21045        .unwrap();
21046        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
21047        let assert_worker = || {
21048            assert_eq!(
21049                crate::worker::current_job(&home).unwrap().unwrap().id,
21050                job.id
21051            );
21052            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
21053            assert_eq!(
21054                crate::worker::helper_executable(&home).unwrap(),
21055                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
21056            );
21057            assert!(
21058                crate::updater::Checker::new(&crate::config::Update {
21059                    mode: crate::config::UpdateMode::Notify,
21060                    interval: None,
21061                    auto_restart: false,
21062                })
21063                .is_none(),
21064                "the driver remains pinned, even while tests are standalone"
21065            );
21066        };
21067        assert_worker();
21068        let repo = home.join("repo");
21069        std::fs::create_dir_all(&repo).unwrap();
21070        init_repo(&repo);
21071        // Keep storage, seat/control context, containment and tool lookup.
21072        // No inherited values (or credentials) are printed on success.
21073        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
21074            test -z "${MAGI_WORKER_GENERATION+x}" &&
21075            test -z "${MAGI_WORKER_HOME+x}" &&
21076            test -z "${MAGI_WORKER_EXE+x}" &&
21077            test "$MAGI_HOME/repo" -ef . &&
21078            test "$MAGI_RUN" = verification-parent &&
21079            test "$MAGI_NODE" = gate &&
21080            test "$MAGI_SEAT" = worker-fixture &&
21081            test -n "$MAGI_PROCESS_OWNER" &&
21082            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
21083            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
21084            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
21085            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
21086        // Negative control: an ordinary child inherits the contamination.
21087        assert!(
21088            !std::process::Command::new("sh")
21089                .args(["-c", probe])
21090                .current_dir(&repo)
21091                .status()
21092                .unwrap()
21093                .success()
21094        );
21095        let unrelated_home = tempfile::tempdir().unwrap();
21096        assert!(
21097            crate::worker::current_job(unrelated_home.path()).is_err(),
21098            "an inherited worker id cannot be resolved in a unit fixture's home"
21099        );
21100        let mut runner = runner_at(RunStatus::Gating);
21101        runner.state.repo = repo.clone();
21102        runner.state.config.disk.min_free_bytes = 0;
21103        runner.state.config.coordination = crate::config::test_coordination();
21104        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
21105        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
21106        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
21107        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
21108        let winner = Candidate {
21109            index: 0,
21110            label: 'A',
21111            agent: "fixture".into(),
21112            branch: "main".into(),
21113            worktree: repo.clone(),
21114            summary: String::new(),
21115            stat: String::new(),
21116            files: 0,
21117            commits: 0,
21118            empty: false,
21119            failed: None,
21120            verified_noop: None,
21121            duration_ms: 0,
21122            folded: false,
21123        };
21124        let (e2e, retried, pids) = run_e2e_with_retry(
21125            &mut runner.state,
21126            &["sh".into(), "-c".into()],
21127            &[format!("{probe} && printf initial")],
21128            &repo,
21129            Duration::from_secs(10),
21130            "initial",
21131            None,
21132        )
21133        .await;
21134        assert!(!retried && pids.is_empty());
21135        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
21136        assert_eq!(e2e[0].output_tail, "initial");
21137        let gate = runner.run_gate(&winner).await.unwrap();
21138        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
21139        // An interrupted repair must recheck e2e from saved state. Landing
21140        // reuses that state and must run both pre-gate and gate again.
21141        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
21142            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
21143            passed: false,
21144            outcomes: Vec::new(),
21145        });
21146        runner.state.save().unwrap();
21147        let mut resumed = RunState::load(&runner.state.id).unwrap();
21148        let wait = crate::repo_lock::WaitPolicy::default();
21149        assert!(matches!(
21150            check_gate_repair_e2e(&mut resumed, &winner, &wait)
21151                .await
21152                .unwrap(),
21153            GateFix::Retry
21154        ));
21155        let saved = RunState::load(&resumed.id).unwrap();
21156        let repair = saved.gate_repair_e2e.unwrap();
21157        assert!(repair.passed);
21158        assert_eq!(repair.outcomes.len(), 1);
21159        assert!(repair.outcomes[0].ok());
21160        let landing = run_landing_gate(&mut resumed, &winner, &wait)
21161            .await
21162            .unwrap();
21163        assert_eq!(landing.len(), 1);
21164        assert!(landing[0].ok(), "{landing:?}");
21165        assert_eq!(
21166            std::fs::read_to_string(repo.join("phases")).unwrap(),
21167            "pre\ngate\ne2e\npre\ngate\n"
21168        );
21169        assert_worker();
21170    }
21171
21172    #[cfg(unix)]
21173    #[tokio::test]
21174    async fn verification_keeps_the_panic_and_the_full_output() {
21175        crate::run::pin_test_home();
21176        let tmp = tempfile::tempdir().unwrap();
21177        let mut config = Config::default();
21178        config.disk.min_free_bytes = 0;
21179        let mut state = RunState::new(
21180            tmp.path().to_owned(),
21181            "main".into(),
21182            "base".into(),
21183            "verification evidence".into(),
21184            config,
21185        );
21186        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
21187                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
21188                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
21189        let mut names = Vec::new();
21190        for _ in 0..2 {
21191            let (outcomes, _) = run_commands(
21192                &mut state,
21193                "gate",
21194                "gate",
21195                0,
21196                &["sh".into(), "-c".into()],
21197                &[script.into()],
21198                tmp.path(),
21199                Duration::from_secs(60),
21200            )
21201            .await;
21202            let tail = &outcomes[0].output_tail;
21203            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
21204            assert!(tail.contains("boom assertion"));
21205            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
21206            let line = tail.lines().next().unwrap();
21207            names.push(line.to_owned());
21208        }
21209        assert_ne!(names[0], names[1], "the second run must not overwrite");
21210        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
21211        assert!(full.len() > 500_000 && full.contains("boom assertion"));
21212        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
21213
21214        // Storage failures are evidence too; preserve the command's verdict.
21215        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
21216        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
21217        let (outcomes, _) = run_commands(
21218            &mut state,
21219            "gate",
21220            "gate",
21221            0,
21222            &["sh".into(), "-c".into()],
21223            &["echo failure-output; exit 7".into()],
21224            tmp.path(),
21225            Duration::from_secs(10),
21226        )
21227        .await;
21228        assert_eq!(outcomes[0].code, Some(7));
21229        assert!(state.events.iter().any(|e| {
21230            e.node == "gate" && e.message.contains("could not save the full gate output:")
21231        }));
21232    }
21233
21234    #[cfg(unix)]
21235    #[tokio::test]
21236    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
21237        crate::run::pin_test_home();
21238        let tmp = tempfile::tempdir().unwrap();
21239        let mut config = Config::default();
21240        config.disk.min_free_bytes = 0;
21241        let mut state = RunState::new(
21242            tmp.path().to_owned(),
21243            "main".into(),
21244            "base".into(),
21245            "verification evidence".into(),
21246            config,
21247        );
21248        let (outcomes, pids) = run_commands(
21249            &mut state,
21250            "gate",
21251            "gate",
21252            1,
21253            &["sh".into(), "-c".into()],
21254            &[
21255                "printf before-timeout; exec sleep 30".into(),
21256                "kill -TERM $$".into(),
21257                "exit 7".into(),
21258                "printf success".into(),
21259            ],
21260            tmp.path(),
21261            Duration::from_secs(1),
21262        )
21263        .await;
21264        wait_for_timed_out_children_to_die(&pids).await;
21265        assert_eq!(
21266            pids.len(),
21267            1,
21268            "unleased timeout retains its death-check PID"
21269        );
21270        assert_eq!(outcomes.len(), 4);
21271        assert_eq!(outcomes[0].code, None);
21272        assert!(outcomes[0].output_tail.contains("before-timeout"));
21273        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
21274        assert_eq!(outcomes[1].code, None);
21275        assert!(outcomes[1].output_tail.contains("command terminated:"));
21276        assert!(!outcomes[1].output_tail.contains("timed out"));
21277        assert_eq!(outcomes[2].code, Some(7));
21278        assert!(!outcomes[2].output_tail.contains("timed out"));
21279        assert_eq!(outcomes[3].code, Some(0));
21280        assert_eq!(outcomes[3].output_tail, "success");
21281        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
21282    }
21283
21284    /// The addendum's second gap: a `verify.gate` command running for real
21285    /// wall-clock time had nothing at all to show for it in `active` before
21286    /// `run_commands` learned to record it — a run could sit in `Gating` for
21287    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
21288    /// what was actually happening. Proven with a genuinely still-running
21289    /// command, not just a before/after check on the final state: a poller
21290    /// task reads the same `run.json` `gate()` is writing, the same way the
21291    /// phone or `magi show` would, while the shell command is still blocked
21292    /// on its own release marker.
21293    #[tokio::test]
21294    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
21295        crate::run::pin_test_home();
21296
21297        let tmp = tempfile::tempdir().expect("tempdir");
21298        let repo = tmp.path().join("repo");
21299        std::fs::create_dir_all(&repo).unwrap();
21300        init_repo(&repo);
21301
21302        let mut config = Config::default();
21303        config.disk.min_free_bytes = 0;
21304        config.verify.gate = vec![
21305            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
21306             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
21307                .to_owned(),
21308        ];
21309
21310        let mut state = RunState::new(
21311            repo.clone(),
21312            "main".to_owned(),
21313            "deadbeef".to_owned(),
21314            "task".to_owned(),
21315            config,
21316        );
21317        let run_id = state.id.clone();
21318        state.candidates = vec![Candidate {
21319            index: 0,
21320            label: 'A',
21321            agent: "alpha".to_owned(),
21322            branch: "does-not-exist".to_owned(),
21323            worktree: repo.clone(),
21324            summary: String::new(),
21325            stat: String::new(),
21326            files: 0,
21327            commits: 0,
21328            empty: false,
21329            failed: None,
21330            verified_noop: None,
21331            duration_ms: 0,
21332            folded: false,
21333        }];
21334        state.tally = Some(Tally {
21335            first_choice: BTreeMap::from([('A', 1)]),
21336            borda: BTreeMap::new(),
21337            winner: 'A',
21338            rankings: 1,
21339            unanimous_initial: true,
21340            deliberated: false,
21341            changed_votes: 0,
21342            unanimous_final: true,
21343            tie_break: None,
21344            judges: 0,
21345            present: 0,
21346            quorum: 0,
21347            met_quorum: true,
21348            uncontested: Some("only candidate A produced a change".to_owned()),
21349        });
21350        state.reviews = vec![ReviewRound {
21351            round: 1,
21352            head: "deadbeef".to_owned(),
21353            verified_head: None,
21354            verified_at: None,
21355            reviews: Vec::new(),
21356            e2e: Vec::new(),
21357            fix: None,
21358            blocking: 0,
21359            answered: 0,
21360            expected: 0,
21361            clean: true,
21362            verify_retried: false,
21363            e2e_deferred: false,
21364            e2e_defer_reason: None,
21365            progressed: false,
21366            vote_split: false,
21367            reconsideration: Vec::new(),
21368            verdict: None,
21369        }];
21370
21371        let mut runner = Runner {
21372            state,
21373            roles: ResolvedRoles {
21374                implementers: Vec::new(),
21375                judges: Vec::new(),
21376                reviewers: Vec::new(),
21377                fixer: None,
21378                conductor: conductor(),
21379                implementer_roster: Vec::new(),
21380                judge_roster: Vec::new(),
21381                reviewer_roster: Vec::new(),
21382            },
21383            sem: Arc::new(Semaphore::new(1)),
21384            pause: Pause::new(),
21385            interrupt: Pause::new(),
21386        };
21387
21388        let started_marker = repo.join("started.marker");
21389        let release_marker = repo.join("release.marker");
21390        let poller = tokio::spawn(async move {
21391            // Bounded so a regression that never records the task entry
21392            // fails this test in seconds instead of hanging the suite —
21393            // the same shape `a_park_requested_while_a_seat_is_mid_call_
21394            // does_not_cut_it_short` uses for the same reason.
21395            for _ in 0..100 {
21396                if started_marker.exists()
21397                    && let Ok(s) = crate::run::RunState::load(&run_id)
21398                    && let Some(a) = s.active.get("gate")
21399                {
21400                    std::fs::write(&release_marker, b"go").expect("release marker");
21401                    return Some(a.clone());
21402                }
21403                tokio::time::sleep(Duration::from_millis(50)).await;
21404            }
21405            None
21406        });
21407
21408        runner.gate().await.expect("gate");
21409        let captured = poller.await.expect("poller task");
21410        let captured = captured.expect(
21411            "the poller never saw a `gate` task entry in run.json while the command was \
21412             still blocked on its own release marker",
21413        );
21414
21415        assert_eq!(captured.task.as_deref(), Some("gate"));
21416        assert_eq!(captured.node, "gate");
21417        assert_eq!(captured.index, Some(1));
21418        assert_eq!(captured.total, Some(1));
21419        assert!(
21420            captured
21421                .command
21422                .as_deref()
21423                .is_some_and(|c| c.contains("started.marker")),
21424            "{captured:?}"
21425        );
21426
21427        assert!(
21428            runner.state.active.is_empty(),
21429            "the entry must be cleared once the command actually finished: {:?}",
21430            runner.state.active
21431        );
21432        assert!(runner.state.gate_ran);
21433        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
21434    }
21435
21436    /// The hand-off over a blocking finding a reviewer rejected on leaves a
21437    /// record for `land`; one with only a Minor, or no reject, leaves none.
21438    #[tokio::test]
21439    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
21440        use crate::verdict::{Finding, ReviewVote, Severity};
21441        crate::run::pin_test_home();
21442        let tmp = tempfile::tempdir().expect("tempdir");
21443        let repo = tmp.path().join("repo");
21444        std::fs::create_dir_all(&repo).unwrap();
21445        init_repo(&repo);
21446
21447        for (severity, vote, expect) in [
21448            (Severity::Major, ReviewVote::Reject, true),
21449            (Severity::Minor, ReviewVote::Reject, false),
21450            (Severity::Major, ReviewVote::Approve, false),
21451        ] {
21452            let mut round = review_round(false, 1, 1, 1, false, true);
21453            round.reviews = vec![ReviewRecord {
21454                reviewer: 1,
21455                agent: "alpha".to_owned(),
21456                summary: String::new(),
21457                findings: vec![Finding {
21458                    id: "R1-1-1".to_owned(),
21459                    severity,
21460                    file: None,
21461                    line: None,
21462                    title: "t".to_owned(),
21463                    detail: String::new(),
21464                }],
21465                vote: Some(vote),
21466                failed: None,
21467                duration_ms: 0,
21468                attempts: 0,
21469            }];
21470            let mut state = RunState::new(
21471                repo.clone(),
21472                "main".to_owned(),
21473                "deadbeef".to_owned(),
21474                "task".to_owned(),
21475                Config::default(),
21476            );
21477            state.reviews = vec![round];
21478            let mut runner = Runner {
21479                state,
21480                roles: ResolvedRoles {
21481                    implementers: Vec::new(),
21482                    judges: Vec::new(),
21483                    reviewers: Vec::new(),
21484                    fixer: None,
21485                    conductor: conductor(),
21486                    implementer_roster: Vec::new(),
21487                    judge_roster: Vec::new(),
21488                    reviewer_roster: Vec::new(),
21489                },
21490                sem: Arc::new(Semaphore::new(1)),
21491                pause: Pause::new(),
21492                interrupt: Pause::new(),
21493            };
21494            let shell = runner.state.config.shell();
21495            runner
21496                .stop_reviewing("round budget spent", &shell, &repo)
21497                .await
21498                .expect("stop_reviewing");
21499            assert_eq!(runner.state.status, RunStatus::Gating);
21500            assert_eq!(
21501                runner.state.contested_handoff.is_some(),
21502                expect,
21503                "{severity:?} + {vote:?}"
21504            );
21505        }
21506    }
21507
21508    /// The shape the incident this whole fix responds to actually had: the
21509    /// round budget spent, the last round's own e2e blocked on the shared
21510    /// build cache (held here by a live pid — this test process — exactly
21511    /// `cache`'s own unit tests' pattern for "another owner, still alive"
21512    /// without forking a process). `stop_reviewing` must retry it — not
21513    /// silently leave the round looking untouched (the catch-up-only half of
21514    /// the bug), and not read the contention as a red `e2e` and block the
21515    /// run on it (the other half). Called directly, the same way
21516    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
21517    /// above exercises `gate`, so this never needs a real cargo build to
21518    /// reach: the lease is never released, so `with_cache_lease` never gets
21519    /// past acquiring it into anything that would need a real workspace.
21520    #[tokio::test]
21521    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
21522        crate::run::pin_test_home();
21523        let home = crate::run::home();
21524
21525        let tmp = tempfile::tempdir().expect("tempdir");
21526        let repo = tmp.path().join("repo");
21527        std::fs::create_dir_all(&repo).unwrap();
21528        init_repo(&repo);
21529        let head = crate::git::rev_parse(&repo, "HEAD")
21530            .await
21531            .expect("rev-parse");
21532        // Unique to this test, so holding its lease cannot collide with
21533        // another test sharing the same process-wide `home`.
21534        let cache_dir = tmp.path().join("target");
21535
21536        let mut config = Config::default();
21537        config.verify.e2e = vec![format!(
21538            "CARGO_TARGET_DIR='{}' test -f README.md",
21539            cache_dir.display()
21540        )];
21541        config.graph.review_rounds = 1;
21542        // Bounded so a regression that does start waiting fails the test in
21543        // seconds, not hangs it.
21544        config.graph.timeout_verify = Some(2);
21545        config.graph.timeout_resource_wait = 2;
21546
21547        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21548        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21549            .expect("no io error acquiring directly")
21550        {
21551            crate::cache::AcquireOutcome::Acquired(g) => g,
21552            crate::cache::AcquireOutcome::Busy(b) => {
21553                panic!("expected the direct acquire to win the lease first: {b:?}")
21554            }
21555        };
21556
21557        let mut state = RunState::new(
21558            repo.clone(),
21559            "main".to_owned(),
21560            head.clone(),
21561            "task".to_owned(),
21562            config,
21563        );
21564        state.candidates = vec![Candidate {
21565            index: 0,
21566            label: 'A',
21567            agent: "alpha".to_owned(),
21568            branch: "does-not-exist".to_owned(),
21569            worktree: repo.clone(),
21570            summary: String::new(),
21571            stat: String::new(),
21572            files: 0,
21573            commits: 0,
21574            empty: false,
21575            failed: None,
21576            verified_noop: None,
21577            duration_ms: 0,
21578            folded: false,
21579        }];
21580        state.tally = Some(Tally {
21581            first_choice: BTreeMap::from([('A', 1)]),
21582            borda: BTreeMap::new(),
21583            winner: 'A',
21584            rankings: 1,
21585            unanimous_initial: true,
21586            deliberated: false,
21587            changed_votes: 0,
21588            unanimous_final: true,
21589            tie_break: None,
21590            judges: 0,
21591            present: 0,
21592            quorum: 0,
21593            met_quorum: true,
21594            uncontested: Some("only candidate A produced a change".to_owned()),
21595        });
21596        // The round budget's last round, deferred: `needs_catchup_run`'s
21597        // other trigger. `stop_reviewing`'s retry machinery must treat this
21598        // exactly like a resource-blocked attempt once it actually runs.
21599        state.reviews = vec![ReviewRound {
21600            round: 1,
21601            head: head.clone(),
21602            verified_head: None,
21603            verified_at: None,
21604            reviews: Vec::new(),
21605            e2e: Vec::new(),
21606            fix: None,
21607            blocking: 1,
21608            answered: 1,
21609            expected: 1,
21610            clean: false,
21611            verify_retried: false,
21612            e2e_deferred: true,
21613            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
21614            progressed: false,
21615            vote_split: false,
21616            reconsideration: Vec::new(),
21617            verdict: None,
21618        }];
21619
21620        let mut runner = Runner {
21621            state,
21622            roles: ResolvedRoles {
21623                implementers: Vec::new(),
21624                judges: Vec::new(),
21625                reviewers: Vec::new(),
21626                fixer: None,
21627                conductor: conductor(),
21628                implementer_roster: Vec::new(),
21629                judge_roster: Vec::new(),
21630                reviewer_roster: Vec::new(),
21631            },
21632            sem: Arc::new(Semaphore::new(1)),
21633            pause: Pause::new(),
21634            interrupt: Pause::new(),
21635        };
21636
21637        let shell = runner.state.config.shell();
21638        runner
21639            .stop_reviewing("round budget spent", &shell, &repo)
21640            .await
21641            .expect("stop_reviewing");
21642
21643        let last = runner.state.reviews.last().expect("round record");
21644        assert_eq!(
21645            last.e2e_status(),
21646            E2eStatus::ResourceBlocked,
21647            "the shared cache is still held; the attempt must read as blocked, not deferred or \
21648             failed: {last:?}"
21649        );
21650        assert_eq!(
21651            last.verified_head.as_deref(),
21652            Some(head.as_str()),
21653            "which commit this attempt targeted is known even though nothing finished checking \
21654             it"
21655        );
21656        let first_attempt_at = last
21657            .verified_at
21658            .expect("when this attempt ran is known too");
21659        assert_ne!(
21660            runner.state.status,
21661            RunStatus::Blocked,
21662            "contention is evidence about the machine, not the patch — it must not settle the \
21663             run as blocked: {:?}",
21664            runner.state.status
21665        );
21666        assert!(
21667            !runner
21668                .state
21669                .events
21670                .iter()
21671                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
21672            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
21673            runner.state.events
21674        );
21675
21676        // The cache is still held: a later reentry must retry the same
21677        // round's verification again — not leave it looking exactly as
21678        // untouched as the first blocked attempt, which is indistinguishable
21679        // from never having tried again at all.
21680        runner
21681            .stop_reviewing("round budget spent", &shell, &repo)
21682            .await
21683            .expect("stop_reviewing retry");
21684        assert_eq!(
21685            runner.state.reviews.len(),
21686            1,
21687            "no new round was started: {:?}",
21688            runner.state.reviews
21689        );
21690        let last = runner.state.reviews.last().expect("round record");
21691        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
21692        assert!(
21693            last.verified_at.expect("still known") > first_attempt_at,
21694            "a second reentry must be a fresh attempt, not a stale copy of the first"
21695        );
21696        assert_ne!(runner.state.status, RunStatus::Blocked);
21697
21698        held.release();
21699    }
21700
21701    #[cfg(unix)]
21702    #[tokio::test]
21703    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
21704        ask_test_home();
21705        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
21706            let tmp = tempfile::tempdir().unwrap();
21707            let repo = tmp.path().join("repo");
21708            std::fs::create_dir(&repo).unwrap();
21709            init_repo(&repo);
21710            let run_git = |args: &[&str]| {
21711                let out = std::process::Command::new("git")
21712                    .args(args)
21713                    .current_dir(&repo)
21714                    .output()
21715                    .unwrap();
21716                assert!(
21717                    out.status.success(),
21718                    "{args:?}: {}",
21719                    String::from_utf8_lossy(&out.stderr)
21720                );
21721            };
21722            let origin = tmp.path().join("origin.git");
21723            run_git(&["init", "--bare", origin.to_str().unwrap()]);
21724            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
21725            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
21726            run_git(&["add", "spec.rs"]);
21727            run_git(&["commit", "-m", "initial spec"]);
21728            run_git(&["push", "origin", "main"]);
21729            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
21730            run_git(&["checkout", "-b", "fix/review"]);
21731            std::fs::write(
21732                repo.join("check.rs"),
21733                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
21734            )
21735            .unwrap();
21736            run_git(&["add", "check.rs"]);
21737            run_git(&["commit", "-m", "use spec"]);
21738            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
21739            run_git(&["checkout", "main"]);
21740            std::fs::write(
21741                repo.join("spec.rs"),
21742                "struct AgentSpec { max_turn_secs: u64 }\n",
21743            )
21744            .unwrap();
21745            run_git(&["add", "spec.rs"]);
21746            run_git(&["commit", "-m", "extend spec"]);
21747            run_git(&["push", "origin", "main"]);
21748            run_git(&["checkout", "fix/review"]);
21749
21750            let mut runner = runner_at(RunStatus::Reviewing);
21751            runner.state.repo = repo.clone();
21752            runner.state.base_commit = base;
21753            runner.state.pr_head = Some("fix/review".into());
21754            runner.state.config.graph.review_rounds = max_rounds;
21755            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
21756            runner.state.config.graph.sessions = false;
21757            runner.state.config.disk.min_free_bytes = 0;
21758            runner.state.config.verify.e2e =
21759                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
21760            runner.state.candidates = vec![Candidate {
21761                index: 0,
21762                label: 'A',
21763                agent: "fixture".into(),
21764                branch: "fix/review".into(),
21765                worktree: repo.clone(),
21766                summary: String::new(),
21767                stat: String::new(),
21768                files: 1,
21769                commits: 1,
21770                empty: false,
21771                failed: None,
21772                verified_noop: None,
21773                duration_ms: 0,
21774                folded: false,
21775            }];
21776            runner.state.tally = Some(Tally {
21777                first_choice: BTreeMap::new(),
21778                borda: BTreeMap::new(),
21779                winner: 'A',
21780                rankings: 0,
21781                unanimous_initial: true,
21782                deliberated: false,
21783                changed_votes: 0,
21784                unanimous_final: true,
21785                tie_break: None,
21786                judges: 0,
21787                present: 0,
21788                quorum: 0,
21789                met_quorum: true,
21790                uncontested: Some("review-only".into()),
21791            });
21792            let mut last = review_round(false, 0, 1, 1, false, false);
21793            last.head = reviewed.clone();
21794            last.e2e_deferred = true;
21795            last.e2e[0].resource_blocked = true;
21796            last.e2e[0].output_tail = "waiting for the shared build cache".into();
21797            if max_rounds == 1 {
21798                last.e2e.clear();
21799            }
21800            runner.state.reviews = vec![last];
21801            let calls = tmp.path().join("fix-calls");
21802            let mut fixer = spec("repair");
21803            fixer
21804                .env
21805                .insert("FIX_CALLS".into(), calls.display().to_string());
21806            fixer.command = vec!["sh".into(), "-c".into(), r#"
21807                echo fix >> "$FIX_CALLS"
21808                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
21809                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
21810                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
21811                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
21812            "#.into()];
21813            let mut reviewer = spec("reviewer");
21814            reviewer.command = vec![
21815                "sh".into(),
21816                "-c".into(),
21817                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
21818                    .into(),
21819            ];
21820            runner.roles.fixer = Some(vec![fixer]);
21821            runner.roles.reviewers = vec![reviewer];
21822
21823            runner.sync_to_base().await.unwrap();
21824            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
21825            assert_ne!(rebased, reviewed);
21826            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
21827            if reentry {
21828                // Simulate a crash after catch-up evidence was saved but before
21829                // dispatching the fixer. No dedicated pending-repair field exists.
21830                let shell = runner.state.config.shell();
21831                runner
21832                    .stop_unresolved_review("deferred verification", &shell, &repo)
21833                    .await
21834                    .unwrap();
21835                assert!(runner.catchup_repairable());
21836                assert!(!calls.exists());
21837                runner.state = RunState::load(&runner.state.id).unwrap();
21838                assert!(runner.catchup_repairable());
21839            }
21840            runner.review_loop().await.unwrap();
21841            let first = &runner.state.reviews[0];
21842            assert_eq!(first.head, reviewed);
21843            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
21844            assert!(
21845                first.verify_retried,
21846                "a reproduced build failure gets exactly one retry: {:?}",
21847                first.e2e
21848            );
21849            assert!(first.e2e[0].output_tail.contains("E0063"));
21850            assert!(!runner.catchup_repairable());
21851            if max_rounds == 2 {
21852                assert_eq!(
21853                    runner.state.status,
21854                    RunStatus::Gating,
21855                    "{}",
21856                    crate::report::run(&runner.state)
21857                );
21858                assert_eq!(runner.state.reviews.len(), 2);
21859                assert!(first.fix.as_ref().unwrap().committed);
21860                assert!(runner.state.reviews[1].clean);
21861                assert_eq!(
21862                    runner
21863                        .state
21864                        .reviews
21865                        .iter()
21866                        .filter(|r| r.fix.is_some())
21867                        .count(),
21868                    1
21869                );
21870                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
21871                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
21872            } else {
21873                assert_eq!(runner.state.status, RunStatus::Blocked);
21874                assert!(first.fix.is_none());
21875                assert!(!calls.exists());
21876                let recovery = crate::recovery::assess(
21877                    "prior",
21878                    Some(&runner.state),
21879                    &runner.landing_base(),
21880                    crate::recovery::Relation::Diverged,
21881                );
21882                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
21883                let reasons = recovery.reasons.join("\n");
21884                assert!(
21885                    reasons.contains(&reviewed) && reasons.contains(&rebased),
21886                    "{reasons}"
21887                );
21888                assert!(
21889                    reasons.contains("base sync to")
21890                        && reasons.contains("additional review is required"),
21891                    "{reasons}"
21892                );
21893            }
21894        }
21895    }
21896
21897    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
21898    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
21899    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
21900    /// own top-of-function fast path (`review_conclusion`) correctly reads
21901    /// this shape as `None` rather than guessing `Blocked`, and the loop's
21902    /// own `for` range is empty once the round budget is spent, so
21903    /// `review_loop` must retry the check itself rather than silently doing
21904    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
21905    /// above exercises directly, but through `review_loop`'s own entry point
21906    /// this time, proving the wiring between the two rather than just the
21907    /// retry logic in isolation.
21908    #[tokio::test]
21909    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
21910        crate::run::pin_test_home();
21911        let home = crate::run::home();
21912
21913        let tmp = tempfile::tempdir().expect("tempdir");
21914        let repo = tmp.path().join("repo");
21915        std::fs::create_dir_all(&repo).unwrap();
21916        init_repo(&repo);
21917        let head = crate::git::rev_parse(&repo, "HEAD")
21918            .await
21919            .expect("rev-parse");
21920        let cache_dir = tmp.path().join("target");
21921
21922        let mut config = Config::default();
21923        config.verify.e2e = vec![format!(
21924            "CARGO_TARGET_DIR='{}' test -f README.md",
21925            cache_dir.display()
21926        )];
21927        config.graph.review_rounds = 1;
21928        config.graph.timeout_verify = Some(2);
21929        config.graph.timeout_resource_wait = 2;
21930
21931        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
21932        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
21933            .expect("no io error acquiring directly")
21934        {
21935            crate::cache::AcquireOutcome::Acquired(g) => g,
21936            crate::cache::AcquireOutcome::Busy(b) => {
21937                panic!("expected the direct acquire to win the lease first: {b:?}")
21938            }
21939        };
21940
21941        let mut state = RunState::new(
21942            repo.clone(),
21943            "main".to_owned(),
21944            head.clone(),
21945            "task".to_owned(),
21946            config,
21947        );
21948        state.candidates = vec![Candidate {
21949            index: 0,
21950            label: 'A',
21951            agent: "alpha".to_owned(),
21952            branch: "does-not-exist".to_owned(),
21953            worktree: repo.clone(),
21954            summary: String::new(),
21955            stat: String::new(),
21956            files: 0,
21957            commits: 0,
21958            empty: false,
21959            failed: None,
21960            verified_noop: None,
21961            duration_ms: 0,
21962            folded: false,
21963        }];
21964        state.tally = Some(Tally {
21965            first_choice: BTreeMap::from([('A', 1)]),
21966            borda: BTreeMap::new(),
21967            winner: 'A',
21968            rankings: 1,
21969            unanimous_initial: true,
21970            deliberated: false,
21971            changed_votes: 0,
21972            unanimous_final: true,
21973            tie_break: None,
21974            judges: 0,
21975            present: 0,
21976            quorum: 0,
21977            met_quorum: true,
21978            uncontested: Some("only candidate A produced a change".to_owned()),
21979        });
21980        // The exact shape a prior process's `stop_reviewing` would have left
21981        // on disk: the round budget's last round, a real attempt already
21982        // made and already resource-blocked.
21983        state.reviews = vec![ReviewRound {
21984            round: 1,
21985            head: head.clone(),
21986            verified_head: Some(head.clone()),
21987            verified_at: Some(jiff::Timestamp::now()),
21988            reviews: Vec::new(),
21989            e2e: vec![CommandOutcome {
21990                command: format!(
21991                    "CARGO_TARGET_DIR='{}' test -f README.md",
21992                    cache_dir.display()
21993                ),
21994                code: None,
21995                output_tail: "waiting for the shared build cache".to_owned(),
21996                duration_ms: 0,
21997                resource_blocked: true,
21998            }],
21999            fix: None,
22000            blocking: 1,
22001            answered: 1,
22002            expected: 1,
22003            clean: false,
22004            verify_retried: false,
22005            e2e_deferred: false,
22006            e2e_defer_reason: None,
22007            progressed: false,
22008            vote_split: false,
22009            reconsideration: Vec::new(),
22010            verdict: None,
22011        }];
22012
22013        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
22014        let mut runner = Runner {
22015            state,
22016            roles: ResolvedRoles {
22017                implementers: Vec::new(),
22018                judges: Vec::new(),
22019                reviewers: Vec::new(),
22020                fixer: None,
22021                conductor: conductor(),
22022                implementer_roster: Vec::new(),
22023                judge_roster: Vec::new(),
22024                reviewer_roster: Vec::new(),
22025            },
22026            sem: Arc::new(Semaphore::new(1)),
22027            pause: Pause::new(),
22028            interrupt: Pause::new(),
22029        };
22030
22031        // The lease is still held throughout, so this reentry's own retry is
22032        // also contended — proving `review_loop` actually tried again (not
22033        // that it happened to succeed) is what the timestamp comparison
22034        // below is for.
22035        runner.review_loop().await.expect("review_loop");
22036
22037        assert_eq!(
22038            runner.state.reviews.len(),
22039            1,
22040            "no new round was started on top of the unresolved one: {:?}",
22041            runner.state.reviews
22042        );
22043        let last = &runner.state.reviews[0];
22044        assert_eq!(
22045            last.e2e_status(),
22046            E2eStatus::ResourceBlocked,
22047            "still contended: {last:?}"
22048        );
22049        assert!(
22050            last.verified_at.expect("still known") > first_attempt_at,
22051            "review_loop must have actually retried the check, not left it exactly as found"
22052        );
22053        assert_ne!(
22054            runner.state.status,
22055            RunStatus::Blocked,
22056            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
22057            runner.state.status
22058        );
22059
22060        held.release();
22061    }
22062
22063    #[tokio::test]
22064    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
22065        crate::run::pin_test_home();
22066        for status in [RunStatus::Blocked, RunStatus::Gating] {
22067            let mut runner = runner_at(status);
22068            runner.state.config.graph.land = true;
22069            runner.state.gate_ran = true;
22070            runner.state.merge = Some(MergeOutcome {
22071                mode: MergeMode::Pr,
22072                ok: false,
22073                detail: "the fixer produced no commit".into(),
22074                empty: false,
22075            });
22076            let url = "https://example.invalid/o/r/pull/1";
22077            runner.state.pr = Some(crate::run::PrRecord {
22078                url: url.into(),
22079                number: 1,
22080                state: "open".into(),
22081                checks: "red".into(),
22082                round: 1,
22083                rounds: 1,
22084                red_at_merge: Vec::new(),
22085            });
22086            runner.state.save().unwrap();
22087            // The nonexistent repository makes observation fail before a forge
22088            // command can run. We still must reach land, rather than silently
22089            // return from the graph with its old gate and merge records.
22090            runner.execute_graph().await.unwrap();
22091            assert_eq!(runner.state.status, RunStatus::Blocked);
22092            assert!(runner.state.land_sync_pending);
22093            assert!(runner.state.events.iter().any(|event|
22094                event.node == "land" && event.message == format!("watching {url}")));
22095            assert_eq!(
22096                runner.state.merge.as_ref().unwrap().detail,
22097                "the fixer produced no commit"
22098            );
22099        }
22100    }
22101
22102    #[tokio::test]
22103    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
22104        crate::run::pin_test_home();
22105        let tmp = tempfile::tempdir().expect("tempdir");
22106        let repo = tmp.path().join("repo");
22107        std::fs::create_dir_all(&repo).unwrap();
22108        init_repo(&repo);
22109
22110        let mut config = Config::default();
22111        config.merge.mode = MergeMode::Pr;
22112        config.graph.land = true;
22113        config.graph.land_approval = false;
22114
22115        let mut state = RunState::new(
22116            repo.clone(),
22117            "main".to_owned(),
22118            "deadbeef".to_owned(),
22119            "task".to_owned(),
22120            config,
22121        );
22122        state.candidates = vec![Candidate {
22123            index: 0,
22124            label: 'A',
22125            agent: "alpha".to_owned(),
22126            branch: "does-not-exist".to_owned(),
22127            worktree: repo.clone(),
22128            summary: String::new(),
22129            stat: String::new(),
22130            files: 0,
22131            commits: 0,
22132            empty: false,
22133            failed: None,
22134            verified_noop: None,
22135            duration_ms: 0,
22136            folded: false,
22137        }];
22138        state.tally = Some(Tally {
22139            first_choice: BTreeMap::from([('A', 1)]),
22140            borda: BTreeMap::new(),
22141            winner: 'A',
22142            rankings: 1,
22143            unanimous_initial: true,
22144            deliberated: false,
22145            changed_votes: 0,
22146            unanimous_final: true,
22147            tie_break: None,
22148            judges: 0,
22149            present: 0,
22150            quorum: 0,
22151            met_quorum: true,
22152            uncontested: Some("only candidate A produced a change".to_owned()),
22153        });
22154        state.reviews = vec![ReviewRound {
22155            round: 1,
22156            head: "deadbeef".to_owned(),
22157            verified_head: None,
22158            verified_at: None,
22159            reviews: Vec::new(),
22160            e2e: Vec::new(),
22161            fix: None,
22162            blocking: 0,
22163            answered: 0,
22164            expected: 0,
22165            clean: true,
22166            verify_retried: false,
22167            e2e_deferred: false,
22168            e2e_defer_reason: None,
22169            progressed: false,
22170            vote_split: false,
22171            reconsideration: Vec::new(),
22172            verdict: None,
22173        }];
22174        state.gate = vec![CommandOutcome {
22175            command: "test".to_owned(),
22176            code: Some(0),
22177            output_tail: String::new(),
22178            duration_ms: 0,
22179            resource_blocked: false,
22180        }];
22181        state.gate_ran = true;
22182        // A first pass through `merge` already pushed and opened this pull
22183        // request; `status` is `Landing` because a previous call into `land`
22184        // parked or was interrupted before it reached a terminal outcome.
22185        state.status = RunStatus::Landing;
22186        state.merge = Some(MergeOutcome {
22187            mode: MergeMode::Pr,
22188            ok: true,
22189            detail: "https://example.invalid/x/y/pull/1".to_owned(),
22190            empty: false,
22191        });
22192
22193        // The Landing-resume shortcut calls `run_land` directly rather than
22194        // through `merge`, which is exactly the call site that used to skip
22195        // `settle_questions` - see the fixture below.
22196        ask_test_home();
22197        // A real resume loads a persisted run; driver ownership is checked
22198        // against that record before the Landing shortcut is entered.
22199        state.save().expect("save landing run");
22200        let store = ask::Questions::open();
22201        let q = ask_open_question(&store, &state.id);
22202
22203        let mut runner = Runner {
22204            state,
22205            roles: ResolvedRoles {
22206                implementers: Vec::new(),
22207                judges: Vec::new(),
22208                reviewers: Vec::new(),
22209                fixer: None,
22210                conductor: conductor(),
22211                implementer_roster: Vec::new(),
22212                judge_roster: Vec::new(),
22213                reviewer_roster: Vec::new(),
22214            },
22215            sem: Arc::new(Semaphore::new(1)),
22216            pause: Pause::new(),
22217            interrupt: Pause::new(),
22218        };
22219
22220        // `execute`, not `merge` directly: the Landing-resume shortcut lives
22221        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
22222        // exactly because `review_loop` would otherwise clobber the marker
22223        // first.
22224        runner.execute().await.expect("execute");
22225
22226        assert_eq!(
22227            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
22228            Some("https://example.invalid/x/y/pull/1"),
22229            "reentry must not push again or open a second pull request over the \
22230             one `land` is already watching"
22231        );
22232        assert_ne!(
22233            runner.state.status,
22234            RunStatus::Landing,
22235            "land could not actually reach the fake pull request, so it must \
22236             have given up rather than left the run silently parked forever"
22237        );
22238        // `land` could not reach the fake pull request, so it gave up into
22239        // `Blocked` - still resumable, so the question must not have been
22240        // swept just because this branch now also calls `settle_questions`.
22241        assert_eq!(runner.state.status, RunStatus::Blocked);
22242        assert!(
22243            store.get(&q.id).unwrap().status.open(),
22244            "Blocked is still alive; settle_questions must have been a no-op here"
22245        );
22246    }
22247
22248    fn state_with_round(round: ReviewRound) -> RunState {
22249        let mut s = RunState::new(
22250            PathBuf::from("/repo"),
22251            "main".to_owned(),
22252            "abc1234".to_owned(),
22253            "add retries".to_owned(),
22254            Config::default(),
22255        );
22256        s.reviews = vec![round];
22257        s
22258    }
22259
22260    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
22261        crate::verdict::Finding {
22262            id: id.to_owned(),
22263            severity,
22264            file: None,
22265            line: None,
22266            title: title.to_owned(),
22267            detail: String::new(),
22268        }
22269    }
22270
22271    #[test]
22272    fn pr_body_names_open_findings_and_declined_ones() {
22273        let round = ReviewRound {
22274            round: 2,
22275            head: "deadbee".to_owned(),
22276            verified_head: None,
22277            verified_at: None,
22278            reviews: vec![ReviewRecord {
22279                attempts: 0,
22280                reviewer: 1,
22281                agent: "alpha".to_owned(),
22282                summary: String::new(),
22283                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
22284                vote: None,
22285                failed: None,
22286                duration_ms: 0,
22287            }],
22288            e2e: vec![CommandOutcome {
22289                command: "cargo test".to_owned(),
22290                code: Some(0),
22291                output_tail: String::new(),
22292                duration_ms: 0,
22293                resource_blocked: false,
22294            }],
22295            verify_retried: false,
22296            e2e_deferred: false,
22297            e2e_defer_reason: None,
22298            fix: Some(FixRecord {
22299                agent: "alpha".to_owned(),
22300                addressed: Vec::new(),
22301                rejected: vec![crate::verdict::Rejection {
22302                    id: "R1-1-1".to_owned(),
22303                    why: "not reachable from any caller".to_owned(),
22304                }],
22305                notes: String::new(),
22306                committed: true,
22307                failed: None,
22308                duration_ms: 0,
22309                continuation: None,
22310            }),
22311            blocking: 0,
22312            answered: 1,
22313            expected: 1,
22314            clean: false,
22315            progressed: true,
22316            vote_split: false,
22317            reconsideration: Vec::new(),
22318            verdict: None,
22319        };
22320        let state = state_with_round(round);
22321        let body = pr_message(&state, 'A').body;
22322
22323        assert!(body.contains("add retries"), "the task must still be there");
22324        assert!(body.contains("R2-1-1"), "{body}");
22325        assert!(!body.contains("unused import"), "{body}");
22326        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
22327        assert!(
22328            !body.contains("not reachable from any caller"),
22329            "the reason it was declined: {body}"
22330        );
22331    }
22332
22333    #[test]
22334    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
22335        let round = ReviewRound {
22336            round: 1,
22337            head: "deadbee".to_owned(),
22338            verified_head: None,
22339            verified_at: None,
22340            reviews: vec![ReviewRecord {
22341                attempts: 0,
22342                reviewer: 1,
22343                agent: "alpha".to_owned(),
22344                summary: String::new(),
22345                findings: Vec::new(),
22346                vote: None,
22347                failed: None,
22348                duration_ms: 0,
22349            }],
22350            e2e: Vec::new(),
22351            verify_retried: false,
22352            e2e_deferred: false,
22353            e2e_defer_reason: None,
22354            fix: None,
22355            blocking: 0,
22356            answered: 1,
22357            expected: 1,
22358            clean: true,
22359            progressed: false,
22360            vote_split: false,
22361            reconsideration: Vec::new(),
22362            verdict: None,
22363        };
22364        let state = state_with_round(round);
22365        let body = pr_message(&state, 'A').body;
22366        assert!(!body.contains("Open review findings"), "{body}");
22367        assert!(!body.contains("Declined"), "{body}");
22368    }
22369
22370    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
22371        let mut state = RunState::new(
22372            PathBuf::from("/repo"),
22373            "main".to_owned(),
22374            "abc1234".to_owned(),
22375            instruction.to_owned(),
22376            Config::default(),
22377        );
22378        // These fixtures test execution flow, independent of host disk capacity.
22379        state.config.disk.min_free_bytes = 0;
22380        state.candidates.push(Candidate {
22381            index: 0,
22382            label: 'A',
22383            agent: "alpha".to_owned(),
22384            branch: "magi/x/A".to_owned(),
22385            worktree: PathBuf::from("/wt"),
22386            summary: summary.to_owned(),
22387            stat: String::new(),
22388            files: 1,
22389            commits: 1,
22390            empty: false,
22391            failed: None,
22392            verified_noop: None,
22393            folded: false,
22394            duration_ms: 0,
22395        });
22396        state
22397    }
22398
22399    fn review_state(subjects: &[&str]) -> RunState {
22400        let mut state = state_with_summary(
22401            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
22402            "",
22403        );
22404        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22405        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
22406        state
22407    }
22408
22409    /// A branch rebased onto a main that moved past the recorded
22410    /// `base_commit` is titled from its own first commit, never main's.
22411    #[tokio::test]
22412    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
22413        ask_test_home();
22414        let tmp = tempfile::tempdir().unwrap();
22415        let repo = tmp.path().join("repo");
22416        std::fs::create_dir_all(&repo).unwrap();
22417        init_repo(&repo);
22418        let origin = tmp.path().join("origin.git");
22419        let g = |dir: &Path, args: &[&str]| {
22420            let out = std::process::Command::new("git")
22421                .args(args)
22422                .current_dir(dir)
22423                .quiet()
22424                .output()
22425                .expect("spawn git");
22426            assert!(
22427                out.status.success(),
22428                "git {args:?}: {}",
22429                String::from_utf8_lossy(&out.stderr)
22430            );
22431        };
22432        g(
22433            tmp.path(),
22434            &[
22435                "clone",
22436                "--bare",
22437                "-q",
22438                repo.to_str().unwrap(),
22439                origin.to_str().unwrap(),
22440            ],
22441        );
22442        g(
22443            &repo,
22444            &["remote", "add", "origin", origin.to_str().unwrap()],
22445        );
22446        let c1 = git::rev_parse(&repo, "main").await.unwrap();
22447
22448        // Main moves on; the branch is built on top of the new main.
22449        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
22450        g(&repo, &["add", "-A"]);
22451        g(
22452            &repo,
22453            &["commit", "-q", "-m", "chore(deps): update a crate"],
22454        );
22455        g(&repo, &["push", "-q", "origin", "main"]);
22456        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
22457        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22458        g(&repo, &["add", "-A"]);
22459        g(
22460            &repo,
22461            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
22462        );
22463        g(&repo, &["checkout", "-q", "main"]);
22464
22465        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
22466        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
22467        // Without a readable tracking ref the recorded base's merge base is used.
22468        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
22469        assert_eq!(fallback, c1);
22470
22471        let mut state = review_state(&[
22472            "chore(deps): update a crate",
22473            "fix(daemon): apply a chosen action",
22474        ]);
22475        state.repo = repo.clone();
22476        state.base_branch = "main".to_owned();
22477        state.base_commit = c1;
22478        refresh_reviewed_commits(&mut state, "feat/own").await;
22479        assert_eq!(
22480            state.reviewed_commits,
22481            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
22482        );
22483        assert_eq!(
22484            review_title(&state).as_deref(),
22485            Some("fix(daemon): apply a chosen action")
22486        );
22487        assert_eq!(
22488            leaked_subjects(&state, "feat/own").await,
22489            Some(vec!["chore(deps): update a crate".to_owned()])
22490        );
22491        // A stale tracking ref is still used when the fetch fails, but the
22492        // leak list is withheld.
22493        state.config.merge.remote = "nowhere".to_owned();
22494        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
22495    }
22496
22497    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
22498    /// with `origin/main` one commit behind it, and a summarizing agent that
22499    /// prints whatever `reply` holds.
22500    struct SummaryFixture {
22501        _tmp: tempfile::TempDir,
22502        repo: PathBuf,
22503        reply: PathBuf,
22504        runner: Runner,
22505    }
22506
22507    fn summary_fixture() -> SummaryFixture {
22508        ask_test_home();
22509        let tmp = tempfile::tempdir().unwrap();
22510        let repo = tmp.path().join("repo");
22511        std::fs::create_dir_all(&repo).unwrap();
22512        init_repo(&repo);
22513        let origin = tmp.path().join("origin.git");
22514        let g = |args: &[&str]| {
22515            let out = std::process::Command::new("git")
22516                .args(args)
22517                .current_dir(&repo)
22518                .quiet()
22519                .output()
22520                .expect("spawn git");
22521            assert!(
22522                out.status.success(),
22523                "git {args:?}: {}",
22524                String::from_utf8_lossy(&out.stderr)
22525            );
22526        };
22527        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
22528        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
22529        g(&["push", "-q", "origin", "main"]);
22530        g(&["checkout", "-q", "-b", "feat/own"]);
22531        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
22532        g(&["add", "-A"]);
22533        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
22534        let reply = tmp.path().join("reply.txt");
22535        std::fs::write(
22536            &reply,
22537            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
22538        )
22539        .unwrap();
22540        let mut runner = runner_at(RunStatus::Gating);
22541        let mut state = review_state(&["feat(own): add the own file"]);
22542        state.repo = repo.clone();
22543        state.base_branch = "main".to_owned();
22544        state.candidates[0].branch = "feat/own".to_owned();
22545        state.candidates[0].worktree = repo.clone();
22546        let mut author = spec("alpha");
22547        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
22548        author
22549            .env
22550            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
22551        state.config.agents = vec![author];
22552        runner.state = state;
22553        SummaryFixture {
22554            _tmp: tmp,
22555            repo,
22556            reply,
22557            runner,
22558        }
22559    }
22560
22561    async fn posted(f: &mut SummaryFixture) -> PrMessage {
22562        let winner = f.runner.state.candidates[0].clone();
22563        f.runner
22564            .guarded_pr_message(&winner, None, true)
22565            .await
22566            .unwrap()
22567            .expect("an English description never parks")
22568    }
22569
22570    fn clean_round_with_minor_findings() -> ReviewRound {
22571        let mut round = passing_round(2, None);
22572        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
22573        round.reviews = vec![ReviewRecord {
22574            attempts: 0,
22575            reviewer: 1,
22576            agent: "alpha".into(),
22577            summary: "内部レビュー".into(),
22578            findings: vec![
22579                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
22580                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
22581            ],
22582            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
22583            failed: None,
22584            duration_ms: 0,
22585        }];
22586        round
22587    }
22588
22589    #[tokio::test]
22590    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
22591        for review_only in [false, true] {
22592            let mut f = summary_fixture();
22593            if !review_only {
22594                f.runner.state.candidates[0].agent = "alpha".into();
22595            }
22596            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
22597            assert!(f.runner.state.reviews[0].clean);
22598            assert!(f.runner.state.reviews[0].fix.is_none());
22599            assert!(f.runner.state.open_findings().is_empty());
22600            assert_eq!(
22601                crate::pr_outcome::required_finding_ids(&f.runner.state),
22602                ["R2-1-1", "R2-1-2"]
22603            );
22604            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22605            let evidence = crate::pr_outcome::summary_evidence(
22606                &f.runner.state,
22607                crate::pr_outcome::FinalHead::At(&head),
22608            );
22609            assert!(
22610                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
22611            );
22612            // The ordinary reply omits both IDs: no cached description is adopted.
22613            let omitted = posted(&mut f).await;
22614            assert!(omitted.body.contains("No description"));
22615            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
22616            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
22617            assert!(f.runner.state.change_summary.is_none());
22618            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
22619            let described = posted(&mut f).await;
22620            assert!(described.body.contains("improve documentation for callers"));
22621            assert!(described.body.contains("remaining edge case"));
22622            assert!(described.body.contains("## Open review findings"));
22623            assert!(!described.body.contains("未修正"));
22624        }
22625    }
22626
22627    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
22628        let mut f = summary_fixture();
22629        if !review_only {
22630            f.runner.state.candidates[0].agent = "alpha".into();
22631        }
22632        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22633        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
22634        fixed
22635            .fix
22636            .as_mut()
22637            .unwrap()
22638            .rejected
22639            .push(crate::verdict::Rejection {
22640                id: "R1-1-1".into(),
22641                why: "呼び出されないコードのため辞退".into(),
22642            });
22643        let mut clean = clean_round_with_minor_findings();
22644        clean.verified_head = Some(head.clone());
22645        f.runner.state.reviews = vec![fixed, clean];
22646        ok_gate(&mut f.runner.state, Some(&head));
22647        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
22648        f.runner.state.config.agents.push(scripted("beta", reply));
22649        f.runner.state.config.roles.language_judge =
22650            Some(crate::config::AgentChoice::One("beta".into()));
22651        f
22652    }
22653
22654    #[tokio::test]
22655    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
22656        for review_only in [false, true] {
22657            let mut f = rewrite_fixture_with_findings(review_only,
22658                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
22659            let m = posted(&mut f).await;
22660            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22661            let expected =
22662                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
22663            assert!(m.body.contains(&expected));
22664            assert!(m.body.contains("## Review rounds"));
22665            assert!(
22666                m.body
22667                    .contains("Gate (`verify.gate`): passed on the final commit")
22668            );
22669            assert!(
22670                m.body
22671                    .contains("Tests (`verify.e2e`): passed on the final commit")
22672            );
22673            let c = f.runner.state.change_summary.as_ref().unwrap();
22674            assert!(
22675                m.body
22676                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
22677            );
22678            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22679            assert!(crate::pr_outcome::covers_required_findings(
22680                &f.runner.state,
22681                &m.body
22682            ));
22683            assert!(m.body.contains("documentation remains incomplete"));
22684            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
22685            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
22686            assert!(f.runner.state.github_text.is_none());
22687        }
22688    }
22689
22690    #[tokio::test]
22691    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
22692        for review_only in [false, true] {
22693            for reply in [
22694                "TITLE: fix: public description\nA short English description.",
22695                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
22696            ] {
22697                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
22698                let winner = f.runner.state.candidates[0].clone();
22699                let posted = f
22700                    .runner
22701                    .guarded_pr_message(&winner, None, true)
22702                    .await
22703                    .unwrap();
22704                assert!(posted.is_none());
22705                assert!(f.runner.state.parked);
22706                let withheld = f.runner.state.github_text.as_ref().unwrap();
22707                assert!(
22708                    withheld.rewrite.is_none(),
22709                    "an incomplete rewrite must not be selectable"
22710                );
22711                let q = ask::Questions::open()
22712                    .get(withheld.question.as_ref().unwrap())
22713                    .unwrap();
22714                assert!(
22715                    !q.choices
22716                        .contains(&crate::github_text::USE_REWRITE.to_owned())
22717                );
22718                assert!(
22719                    q.detail.contains("R1-1-1")
22720                        && q.detail.contains("R2-1-1")
22721                        && q.detail.contains("R2-1-2")
22722                );
22723            }
22724        }
22725    }
22726
22727    #[tokio::test]
22728    async fn japanese_reports_stay_local_in_both_public_run_modes() {
22729        for review_only in [false, true] {
22730            let mut f = summary_fixture();
22731            if !review_only {
22732                f.runner.state.candidates[0].agent = "alpha".into();
22733            }
22734            f.runner.state.instruction = "内部タスクの説明".into();
22735            f.runner.state.candidates[0].summary = "内部の候補報告".into();
22736            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22737            f.runner.state.reviews = vec![
22738                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
22739                round_with_notes(2, Some("二回目の内部修正報告です。")),
22740                passing_round(3, Some(&head)),
22741            ];
22742            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
22743            ok_gate(&mut f.runner.state, Some(&head));
22744            let captured = f.repo.join("public-prompt.txt");
22745            let author = &mut f.runner.state.config.agents[0];
22746            author.command = vec![
22747                "sh".into(),
22748                "-c".into(),
22749                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22750            ];
22751            author.env.insert(
22752                "PROMPT_FILE".into(),
22753                captured.to_string_lossy().into_owned(),
22754            );
22755            let m = posted(&mut f).await;
22756            assert!(m.body.contains("Adds `own.txt`"));
22757            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
22758            assert!(m.body.contains("R1-1-1"));
22759            assert!(
22760                m.body
22761                    .contains("Gate (`verify.gate`): passed on the final commit")
22762            );
22763            assert!(
22764                m.body
22765                    .contains("Tests (`verify.e2e`): passed on the final commit")
22766            );
22767            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22768            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
22769            let prompt = std::fs::read_to_string(captured).unwrap();
22770            assert!(prompt.contains(&format!(
22771                "git diff {} {head}",
22772                f.runner.state.change_summary.as_ref().unwrap().base
22773            )));
22774            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
22775            assert_eq!(
22776                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
22777                "二回目の内部修正報告です。"
22778            );
22779        }
22780    }
22781
22782    #[tokio::test]
22783    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
22784        for review_only in [false, true] {
22785            let mut f = summary_fixture();
22786            if !review_only {
22787                f.runner.state.candidates[0].agent = "alpha".into();
22788            }
22789            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
22790            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
22791            std::fs::write(&f.reply, "no usable description").unwrap();
22792            let failed = posted(&mut f).await;
22793            assert!(
22794                failed
22795                    .body
22796                    .contains("No description of what this change does could be generated")
22797            );
22798            assert!(failed.body.is_ascii());
22799            assert!(f.runner.state.github_text.is_none());
22800            assert!(f.runner.state.change_summary.is_none());
22801            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22802            let evidence = crate::pr_outcome::summary_evidence(
22803                &f.runner.state,
22804                crate::pr_outcome::FinalHead::At(&head),
22805            );
22806            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
22807            assert!(
22808                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
22809                    .exists()
22810            );
22811            f.runner.state =
22812                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
22813            std::fs::write(
22814                &f.reply,
22815                "TITLE: feat(own): add the own file\nAdds the committed own file.",
22816            )
22817            .unwrap();
22818            let retried = posted(&mut f).await;
22819            assert!(retried.body.contains("Adds the committed own file."));
22820            assert_eq!(
22821                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
22822                "日本語の修正報告"
22823            );
22824        }
22825    }
22826
22827    #[tokio::test]
22828    async fn same_head_verification_and_findings_invalidate_public_prose() {
22829        let mut f = summary_fixture();
22830        posted(&mut f).await;
22831        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22832        let original = f
22833            .runner
22834            .state
22835            .change_summary
22836            .as_ref()
22837            .unwrap()
22838            .evidence
22839            .clone();
22840        ok_gate(&mut f.runner.state, Some(&head));
22841        std::fs::write(
22842            &f.reply,
22843            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
22844        )
22845        .unwrap();
22846        let verified = posted(&mut f).await;
22847        assert!(verified.body.contains("now verified"));
22848        assert_ne!(
22849            f.runner.state.change_summary.as_ref().unwrap().evidence,
22850            original
22851        );
22852        let mut round = round_with_notes(1, Some("内部報告"));
22853        round.reviews.push(ReviewRecord {
22854            attempts: 0,
22855            reviewer: 1,
22856            agent: "alpha".into(),
22857            summary: "内部レビュー".into(),
22858            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
22859            vote: None,
22860            failed: None,
22861            duration_ms: 0,
22862        });
22863        round
22864            .fix
22865            .as_mut()
22866            .unwrap()
22867            .rejected
22868            .push(crate::verdict::Rejection {
22869                id: "R1-1-1".into(),
22870                why: "呼び出し元から到達できない".into(),
22871            });
22872        f.runner.state.reviews = vec![round];
22873        // A description omitting the required ID is unusable, even with a title.
22874        let omitted = posted(&mut f).await;
22875        assert!(omitted.body.contains("No description"));
22876        assert!(omitted.body.contains("R1-1-1"));
22877        assert!(!omitted.body.contains("到達"));
22878        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
22879        let described = posted(&mut f).await;
22880        assert!(described.body.contains("unreachable from callers"));
22881        assert!(described.body.contains("R1-1-1"));
22882        assert!(!described.body.contains("到達"));
22883        let evidence = f
22884            .runner
22885            .state
22886            .change_summary
22887            .as_ref()
22888            .unwrap()
22889            .evidence
22890            .clone();
22891        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
22892        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
22893        assert!(posted(&mut f).await.body.contains("rationale updated"));
22894        assert_ne!(
22895            f.runner.state.change_summary.as_ref().unwrap().evidence,
22896            evidence
22897        );
22898        // Old serialized summaries have no evidence fingerprint and must regenerate.
22899        f.runner
22900            .state
22901            .change_summary
22902            .as_mut()
22903            .unwrap()
22904            .evidence
22905            .clear();
22906        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
22907        assert!(posted(&mut f).await.body.contains("current specification"));
22908    }
22909
22910    #[tokio::test]
22911    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
22912        let mut f = summary_fixture();
22913        let m = posted(&mut f).await;
22914        assert_eq!(m.title, "feat(own): add the own file");
22915        assert!(
22916            m.body
22917                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
22918            "{}",
22919            m.body
22920        );
22921        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
22922        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
22923        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
22924        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
22925        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
22926        let kept = f.runner.state.change_summary.as_ref().unwrap();
22927        assert_eq!(
22928            (kept.head.as_str(), kept.base.as_str()),
22929            (head.as_str(), base.as_str())
22930        );
22931        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
22932    }
22933
22934    #[tokio::test]
22935    async fn review_only_summary_receives_the_configured_github_language() {
22936        let mut f = summary_fixture();
22937        f.runner.state.config.graph.github_language = "ja".into();
22938        let captured = f.repo.join("summary-prompt.txt");
22939        let author = &mut f.runner.state.config.agents[0];
22940        author.command = vec![
22941            "sh".into(),
22942            "-c".into(),
22943            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
22944        ];
22945        author.env.insert(
22946            "PROMPT_FILE".into(),
22947            captured.to_string_lossy().into_owned(),
22948        );
22949        std::fs::write(
22950            &f.reply,
22951            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
22952        )
22953        .unwrap();
22954        let m = posted(&mut f).await;
22955        let prompt = std::fs::read_to_string(captured).unwrap();
22956        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
22957        assert!(!prompt.contains("in English"));
22958        assert_eq!(m.title, "feat(own): ファイルを追加");
22959        assert!(m.body.contains("own.txt を追加します。"));
22960    }
22961
22962    #[tokio::test]
22963    async fn review_fixes_come_after_the_whole_change_summary() {
22964        let mut f = summary_fixture();
22965        f.runner.state.reviews = vec![
22966            round_with_notes(1, Some("handled the PATH case")),
22967            round_with_notes(2, None),
22968        ];
22969        let m = posted(&mut f).await;
22970        let summary = m.body.find("## Summary").expect("summary");
22971        let fixes = m.body.find("## Review rounds").expect("review rounds");
22972        assert!(summary < fixes, "{}", m.body);
22973        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
22974        assert!(m.body.contains("## Final verification"), "{}", m.body);
22975    }
22976
22977    #[tokio::test]
22978    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
22979        let mut f = summary_fixture();
22980        let first = posted(&mut f).await;
22981        let first_head = f.runner.state.change_summary.clone().unwrap().head;
22982        // The same head reuses the record: a different reply is not read.
22983        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
22984        let again = posted(&mut f).await;
22985        assert_eq!((again.title, again.body), (first.title, first.body));
22986
22987        let g = |args: &[&str]| {
22988            let out = std::process::Command::new("git")
22989                .args(args)
22990                .current_dir(&f.repo)
22991                .quiet()
22992                .output()
22993                .expect("spawn git");
22994            assert!(out.status.success(), "git {args:?}");
22995        };
22996        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
22997        g(&["add", "-A"]);
22998        g(&["commit", "-q", "-m", "fix(own): handle more"]);
22999        let second = posted(&mut f).await;
23000        let kept = f.runner.state.change_summary.clone().unwrap();
23001        assert_ne!(kept.head, first_head);
23002        assert!(second.body.contains("Not read yet."), "{}", second.body);
23003        assert!(!second.body.contains("so the first summary is read"));
23004        assert_eq!(second.title, "feat(own): other");
23005    }
23006
23007    #[tokio::test]
23008    async fn an_unavailable_summary_is_stated_and_never_invented() {
23009        let mut f = summary_fixture();
23010        std::fs::write(&f.reply, "no title line here").unwrap();
23011        let m = posted(&mut f).await;
23012        assert!(
23013            m.body
23014                .contains("No description of what this change does could be generated"),
23015            "{}",
23016            m.body
23017        );
23018        assert!(m.body.contains("internal reports remain in local records"));
23019        assert!(f.runner.state.change_summary.is_none());
23020        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
23021        // A preview asks nobody and claims nothing.
23022        let winner = f.runner.state.candidates[0].clone();
23023        let preview = f
23024            .runner
23025            .guarded_pr_message(&winner, None, false)
23026            .await
23027            .unwrap()
23028            .unwrap();
23029        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
23030    }
23031
23032    #[test]
23033    fn pr_message_review_single_commit_uses_its_subject() {
23034        let state = review_state(&["feat(nats): per-role user"]);
23035        let m = pr_message(&state, 'A');
23036        assert_eq!(m.title, "feat(nats): per-role user");
23037        assert!(!m.body.contains("Review the work already"), "{}", m.body);
23038        assert!(m.body.contains("## Commits under review"), "{}", m.body);
23039    }
23040
23041    #[test]
23042    fn pr_message_review_multi_commit_takes_the_oldest() {
23043        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
23044        let m = pr_message(&state, 'A');
23045        assert_eq!(m.title, "feat: the change");
23046        for s in ["feat: the change", "fix: typo", "fix: again"] {
23047            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
23048        }
23049    }
23050
23051    #[test]
23052    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
23053        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
23054            let mut state = review_state(&[first, "fix: later fixup"]);
23055            state.candidates[0].branch = "機能/ログイン".to_owned();
23056            let m = pr_message(&state, 'A');
23057            assert!(
23058                m.title.starts_with("chore: land candidate A of run"),
23059                "{}",
23060                m.title
23061            );
23062        }
23063    }
23064
23065    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
23066        BranchFacts {
23067            commits: commits
23068                .iter()
23069                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
23070                .collect(),
23071            stat: stat.to_owned(),
23072        }
23073    }
23074
23075    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
23076        let mut r = review_round(true, 0, 1, 1, true, true);
23077        r.round = round;
23078        r.fix = notes.map(|n| FixRecord {
23079            agent: "fixer".to_owned(),
23080            addressed: Vec::new(),
23081            rejected: Vec::new(),
23082            notes: n.to_owned(),
23083            committed: true,
23084            failed: None,
23085            duration_ms: 0,
23086            continuation: None,
23087        });
23088        r
23089    }
23090
23091    #[test]
23092    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
23093        let state = review_state(&["ignored"]);
23094        let f = facts(
23095            &[
23096                (
23097                    "fix(login): resolve PATH on macOS",
23098                    "Login shells skip rc files.",
23099                ),
23100                ("fix: address review", ""),
23101            ],
23102            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
23103        );
23104        let m = pr_message_with(&state, 'A', Some(&f));
23105        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
23106        assert!(
23107            m.body
23108                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
23109            "{}",
23110            m.body
23111        );
23112        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
23113        assert!(m.body.contains("## Diff stat"), "{}", m.body);
23114        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
23115        for banned in [
23116            "Review the work already",
23117            "no task statement",
23118            "Original task",
23119        ] {
23120            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
23121        }
23122        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
23123    }
23124
23125    #[test]
23126    fn pr_message_review_truncates_a_huge_first_commit_body() {
23127        let state = review_state(&["ignored"]);
23128        let f = facts(
23129            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
23130            "s",
23131        );
23132        let m = pr_message_with(&state, 'A', Some(&f));
23133        assert!(m.body.len() < 30_000, "{}", m.body.len());
23134        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
23135        assert!(
23136            m.body.contains("- ... 1 more commit(s)"),
23137            "{}",
23138            m.body.len()
23139        );
23140        assert!(m.body.ends_with("magi:candidate-a\n"));
23141    }
23142
23143    #[test]
23144    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
23145        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
23146        assert_eq!(m.title, "feat: x");
23147        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
23148        assert!(!m.body.contains("Diff stat"), "{}", m.body);
23149        assert!(!m.body.contains("no task statement"), "{}", m.body);
23150    }
23151
23152    #[test]
23153    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
23154        let mut state = review_state(&["日本語の件名"]);
23155        state.candidates[0].branch = "fix/macos-login-path".to_owned();
23156        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
23157        state.candidates[0].branch = "機能/ログイン".to_owned();
23158        assert!(
23159            pr_message(&state, 'A')
23160                .title
23161                .starts_with("chore: land candidate A")
23162        );
23163    }
23164
23165    #[test]
23166    fn pr_message_review_fixes_survive_a_clean_final_round() {
23167        let mut state = review_state(&["feat: x"]);
23168        state.reviews = vec![
23169            round_with_notes(1, Some("handled the PATH case")),
23170            round_with_notes(2, None),
23171        ];
23172        let body = pr_message(&state, 'A').body;
23173        assert!(
23174            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
23175            "{body}"
23176        );
23177        assert!(body.contains("### Round 1"), "{body}");
23178
23179        state.reviews = vec![
23180            round_with_notes(1, Some("first fix")),
23181            round_with_notes(2, Some("")),
23182            round_with_notes(3, Some("second fix")),
23183            round_with_notes(4, None),
23184        ];
23185        let body = pr_message(&state, 'A').body;
23186        assert!(
23187            body.contains("### Round 1") && !body.contains("first fix"),
23188            "{body}"
23189        );
23190        assert!(
23191            body.contains("### Round 3") && !body.contains("second fix"),
23192            "{body}"
23193        );
23194        assert!(body.contains("### Round 2"), "{body}");
23195    }
23196
23197    const FINAL: &str = "abcdef1234567890";
23198
23199    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
23200        let mut r = round_with_notes(round, None);
23201        r.verified_head = verified.map(str::to_owned);
23202        r
23203    }
23204
23205    fn ok_gate(state: &mut RunState, head: Option<&str>) {
23206        state.gate = vec![CommandOutcome {
23207            command: "gate".to_owned(),
23208            code: Some(0),
23209            output_tail: String::new(),
23210            duration_ms: 0,
23211            resource_blocked: false,
23212        }];
23213        state.gate_ran = true;
23214        state.gate_head = head.map(str::to_owned);
23215    }
23216
23217    fn body_at(state: &RunState) -> String {
23218        pr_message_raw(
23219            state,
23220            'A',
23221            None,
23222            crate::pr_outcome::FinalHead::At(FINAL),
23223            ChangeNote::NotAsked,
23224        )
23225        .body
23226    }
23227
23228    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
23229
23230    #[test]
23231    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
23232        let mut state = review_state(&["feat: x"]);
23233        state.reviews = vec![
23234            round_with_notes(1, Some(SANDBOXED)),
23235            passing_round(2, Some(FINAL)),
23236        ];
23237        ok_gate(&mut state, Some(FINAL));
23238        let body = body_at(&state);
23239        assert!(!body.contains("Handled the PATH case."), "{body}");
23240        assert!(!body.contains("could not commit"), "{body}");
23241        assert!(!body.contains("could not run"), "{body}");
23242        assert!(
23243            body.contains("The fix is committed on the branch"),
23244            "{body}"
23245        );
23246        assert!(!body.contains("sandbox"), "{body}");
23247        assert!(
23248            body.contains("Tests (`verify.e2e`): passed on the final commit"),
23249            "{body}"
23250        );
23251        assert!(
23252            body.contains("Gate (`verify.gate`): passed on the final commit"),
23253            "{body}"
23254        );
23255        // The raw report is untouched in the record.
23256        assert!(
23257            state.reviews[0]
23258                .fix
23259                .as_ref()
23260                .unwrap()
23261                .notes
23262                .contains("could not commit")
23263        );
23264    }
23265
23266    #[test]
23267    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
23268        let mut state = review_state(&["feat: x"]);
23269        let mut r1 = round_with_notes(1, Some(SANDBOXED));
23270        r1.fix.as_mut().unwrap().committed = false;
23271        let mut r2 = passing_round(2, Some(FINAL));
23272        r2.e2e[0].code = Some(1);
23273        state.reviews = vec![r1, r2];
23274        let body = body_at(&state);
23275        assert!(!body.contains("could not commit"), "{body}");
23276        assert!(!body.contains("could not run"), "{body}");
23277        assert!(body.contains("This round produced no commit"), "{body}");
23278        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
23279        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
23280        assert!(!body.contains("passed"), "{body}");
23281    }
23282
23283    #[test]
23284    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
23285        let mut state = review_state(&["feat: x"]);
23286        state.reviews = vec![
23287            round_with_notes(1, Some(SANDBOXED)),
23288            passing_round(2, Some("0123456789abcdef")),
23289        ];
23290        ok_gate(&mut state, Some("0123456789abcdef"));
23291        let body = body_at(&state);
23292        assert!(!body.contains("HTTP tests could not run"), "{body}");
23293        // The commit claim is still settled by the structured record.
23294        assert!(!body.contains("could not commit"), "{body}");
23295        assert!(
23296            body.contains("passed on an earlier commit (`0123456`)"),
23297            "{body}"
23298        );
23299        assert!(!body.contains("passed on the final commit"), "{body}");
23300
23301        // A gate recorded before `gate_head` existed is "not recorded".
23302        ok_gate(&mut state, None);
23303        let body = body_at(&state);
23304        assert!(
23305            body.contains("passed on a commit that was not recorded"),
23306            "{body}"
23307        );
23308    }
23309
23310    #[test]
23311    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
23312        let state = state_with_summary(
23313            "add retries to the client",
23314            "TITLE: feat: retries\n\nDid it.",
23315        );
23316        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
23317        assert_eq!(m.title, "feat: retries");
23318        assert!(
23319            m.body.contains("<summary>Original task</summary>"),
23320            "{}",
23321            m.body
23322        );
23323        assert!(!m.body.contains("Commits under review"), "{}", m.body);
23324        assert!(
23325            m.body
23326                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
23327            "{}",
23328            m.body
23329        );
23330    }
23331
23332    #[test]
23333    fn pr_message_review_bounds_a_long_english_subject() {
23334        let long = format!("feat: {}", "word ".repeat(100));
23335        let m = pr_message(&review_state(&[&long]), 'A');
23336        assert!(m.title.starts_with("feat: word"), "{}", m.title);
23337        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23338    }
23339
23340    #[test]
23341    fn pr_message_implementation_run_is_unchanged_by_review_support() {
23342        let state = state_with_summary("add retries\n\ndetails", "- did some things");
23343        let m = pr_message(&state, 'A');
23344        assert_eq!(m.title, "add retries");
23345        assert!(m.body.contains("<summary>Original task</summary>"));
23346        assert!(!m.body.contains("Commits under review"));
23347        assert_eq!(landing_subject_source(&state), state.instruction);
23348    }
23349
23350    #[test]
23351    fn review_run_squash_subject_is_the_change_not_the_prompt() {
23352        let state = review_state(&["feat: the change", "fix: typo"]);
23353        let source = landing_subject_source(&state);
23354        assert_eq!(land::merge_subject("", &source), "feat: the change");
23355        assert_eq!(
23356            land::merge_subject("magi: candidate A (uncommitted work)", &source),
23357            "feat: the change"
23358        );
23359        // An operator's rename still wins.
23360        assert_eq!(
23361            land::merge_subject("feat: renamed by hand", &source),
23362            "feat: renamed by hand"
23363        );
23364        let blank = review_state(&["日本語"]);
23365        assert!(
23366            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
23367        );
23368    }
23369
23370    #[test]
23371    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
23372        let state = review_state(&["feat: the change"]);
23373        let source = landing_subject_source(&state);
23374        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
23375        assert_eq!(
23376            land::merge_subject(landing_title(&state, old), &source),
23377            "feat: the change"
23378        );
23379        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
23380        let task = state_with_summary("add retries", "");
23381        assert_eq!(landing_title(&task, old), old);
23382    }
23383
23384    #[test]
23385    fn pr_message_describes_the_change_not_the_task() {
23386        let state = state_with_summary(
23387            "今回やってほしいこと: results projector を直す",
23388            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
23389        );
23390        let m = pr_message(&state, 'A');
23391        assert_eq!(m.title, "fix(web): batch the runs list reads");
23392        assert!(
23393            m.body.starts_with("## Summary\n\n- reads run.json once"),
23394            "{}",
23395            m.body
23396        );
23397        assert!(!m.body.contains("TITLE:"), "{}", m.body);
23398        let task_at = m.body.find("今回やってほしいこと").unwrap();
23399        let details_at = m.body.find("<details>").unwrap();
23400        assert!(
23401            details_at < task_at,
23402            "the task lives inside <details>: {}",
23403            m.body
23404        );
23405        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
23406        assert!(m.body.contains("magi:candidate-a"));
23407    }
23408
23409    #[test]
23410    fn pr_message_falls_back_to_the_task_without_a_title_line() {
23411        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
23412        let m = pr_message(&state, 'A');
23413        assert_eq!(m.title, "add retries");
23414        assert!(
23415            m.body.contains("## Summary\n\n- did some things"),
23416            "{}",
23417            m.body
23418        );
23419
23420        let none = RunState::new(
23421            PathBuf::from("/repo"),
23422            "main".to_owned(),
23423            "abc1234".to_owned(),
23424            "add retries".to_owned(),
23425            Config::default(),
23426        );
23427        let m = pr_message(&none, 'A');
23428        assert_eq!(m.title, "add retries");
23429        assert!(!m.body.contains("## Summary"), "{}", m.body);
23430    }
23431
23432    #[test]
23433    fn pr_message_refuses_the_candidate_commit_subject() {
23434        for bad in [
23435            "TITLE: magi: candidate A (uncommitted work)",
23436            "TITLE: chore: stuff (uncommitted work)",
23437            "TITLE:   ",
23438        ] {
23439            let state = state_with_summary("add retries", bad);
23440            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
23441        }
23442    }
23443
23444    #[test]
23445    fn pr_message_bounds_a_very_long_task_and_title() {
23446        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
23447        let state = state_with_summary(&long, "- nothing");
23448        let m = pr_message(&state, 'A');
23449        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23450        assert!(!m.title.contains('\n'));
23451
23452        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
23453        let m = pr_message(&state, 'A');
23454        assert!(m.title.starts_with("feat: "));
23455        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
23456        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
23457    }
23458
23459    fn long_title_of(instruction: &str) -> String {
23460        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
23461    }
23462
23463    #[test]
23464    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
23465        let first = "Make the landing path keep a readable title for long tasks";
23466        let line = format!(
23467            "{first}. {}",
23468            "And then keep going with more words ".repeat(20)
23469        );
23470        let t = long_title_of(&line);
23471        assert_eq!(t, first);
23472        assert!(!t.starts_with("chore: land"));
23473    }
23474
23475    #[test]
23476    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
23477        let line = "word ".repeat(200);
23478        let t = long_title_of(&line);
23479        assert!(t.ends_with("word..."), "{t}");
23480        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
23481    }
23482
23483    #[test]
23484    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
23485        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
23486            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
23487        }
23488    }
23489
23490    #[test]
23491    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
23492        let mut state = state_with_summary("再試行を追加する", "- nothing");
23493        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
23494        state.config.graph.github_language = "ja".into();
23495        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
23496    }
23497
23498    #[test]
23499    fn pr_message_title_limit_is_exact() {
23500        let at = "a".repeat(PR_TITLE_MAX);
23501        assert_eq!(long_title_of(&at), at);
23502        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
23503        assert!(over.ends_with("..."), "{over}");
23504        assert_eq!(over.chars().count(), PR_TITLE_MAX);
23505    }
23506
23507    #[test]
23508    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
23509        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
23510        let t = long_title_of(&line);
23511        assert!(t.ends_with("..."), "{t}");
23512        assert!(t.is_ascii(), "{t}");
23513    }
23514
23515    #[test]
23516    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
23517        let line = format!(
23518            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
23519            "plus more filler words ".repeat(20)
23520        );
23521        let t = long_title_of(&line);
23522        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
23523    }
23524
23525    #[test]
23526    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
23527        let line = format!("\n\n# {}", "title words ".repeat(40));
23528        let state = state_with_summary(&line, "- nothing");
23529        let m = pr_message(&state, 'A');
23530        assert!(m.title.starts_with("title words"), "{}", m.title);
23531        assert_eq!(
23532            land::merge_subject(&m.title, &landing_subject_source(&state)),
23533            m.title
23534        );
23535        // An operator's rename wins untouched.
23536        assert_eq!(
23537            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
23538            "feat: renamed by hand"
23539        );
23540    }
23541
23542    #[tokio::test]
23543    async fn github_text_rewrite_is_bounded_and_falls_back() {
23544        crate::run::pin_test_home();
23545        for (reply, accepted) in [
23546            (
23547                "TITLE: fix: retries\nAdd retries for failed requests.",
23548                true,
23549            ),
23550            (
23551                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
23552                false,
23553            ),
23554            (
23555                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
23556                true,
23557            ),
23558            (
23559                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
23560                true,
23561            ),
23562            (
23563                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
23564                true,
23565            ),
23566            (
23567                "TITLE: fix: retries\n日本語の説明です。 token=secret",
23568                false,
23569            ),
23570            ("TITLE: fix: retries\n", false),
23571        ] {
23572            let dir = tempfile::tempdir().unwrap();
23573            let mut runner = runner_at(RunStatus::Gating);
23574            runner.state = state_with_summary(
23575                "add retries",
23576                "TITLE: fix: retries\n日本語の説明を書きます。",
23577            );
23578            runner.state.repo = dir.path().to_owned();
23579            runner.state.candidates[0].worktree = dir.path().to_owned();
23580            let mut author = spec("alpha");
23581            author.command = vec![
23582                "sh".into(),
23583                "-c".into(),
23584                "printf '%s' \"$REWRITE_REPLY\"".into(),
23585            ];
23586            author.env.insert("REWRITE_REPLY".into(), reply.into());
23587            runner.state.config.agents = vec![author];
23588            seed_public_summary(&mut runner);
23589            let winner = runner.state.candidates[0].clone();
23590            let first = runner
23591                .guarded_pr_message(&winner, None, true)
23592                .await
23593                .unwrap();
23594            let message = if accepted {
23595                first.expect("a passing rewrite never asks")
23596            } else {
23597                // Withheld: the run parks on a question and posts nothing yet.
23598                assert!(first.is_none());
23599                assert!(runner.state.parked);
23600                let store = ask::Questions::open();
23601                let asked: Vec<_> = store
23602                    .list()
23603                    .into_iter()
23604                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23605                    .collect();
23606                assert_eq!(asked.len(), 1);
23607                assert!(!asked[0].detail.contains("secret"));
23608                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
23609                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
23610                assert_eq!(
23611                    asked[0]
23612                        .choices
23613                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
23614                    rewrite.is_some()
23615                );
23616                // Silence: the question lapses and the neutral text is used,
23617                // without a second rewrite call or a second question.
23618                store
23619                    .update(&asked[0].id, |q| {
23620                        q.abandon("test");
23621                        Ok(())
23622                    })
23623                    .unwrap();
23624                let again = runner
23625                    .guarded_pr_message(&winner, None, true)
23626                    .await
23627                    .unwrap()
23628                    .expect("an abandoned question falls back");
23629                let n = store
23630                    .list()
23631                    .into_iter()
23632                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23633                    .count();
23634                assert_eq!(n, 1, "asked once per rejected text");
23635                again
23636            };
23637            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
23638            assert!(!message.body.contains("secret"));
23639            assert!(!message.body.contains("prefix"));
23640            assert!(!message.title.contains("secret"));
23641            assert_eq!(
23642                message.body.contains("Add retries for failed requests."),
23643                accepted
23644            );
23645            assert_eq!(
23646                runner.state.seats["github-text-rewrite"].turns, 1,
23647                "only one rewrite invocation"
23648            );
23649            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
23650            if accepted {
23651                assert!(runner.state.github_text.is_none());
23652                assert!(!runner.state.parked);
23653                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
23654                assert!(message.body.contains("## Final verification"));
23655                assert!(message.body.contains("Evidence: final diff"));
23656                assert!(
23657                    message
23658                        .body
23659                        .contains(&format!("magi:run/{}", runner.state.id))
23660                );
23661            }
23662            if !accepted {
23663                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23664                assert!(
23665                    message
23666                        .body
23667                        .contains(&format!("magi:run/{}", runner.state.id))
23668                );
23669            }
23670        }
23671    }
23672
23673    fn scripted(id: &str, reply: &str) -> AgentSpec {
23674        let mut a = spec(id);
23675        a.command = vec![
23676            "sh".into(),
23677            "-c".into(),
23678            "printf '%s' \"$REWRITE_REPLY\"".into(),
23679        ];
23680        a.env.insert("REWRITE_REPLY".into(), reply.into());
23681        a
23682    }
23683
23684    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
23685    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
23686
23687    /// Gate tests inject generated public output, not internal reports.
23688    fn seed_public_summary(runner: &mut Runner) {
23689        let repo = &runner.state.repo;
23690        let g = |args: &[&str]| {
23691            let out = std::process::Command::new("git")
23692                .args(args)
23693                .current_dir(repo)
23694                .quiet()
23695                .output()
23696                .unwrap();
23697            assert!(out.status.success(), "git {args:?}");
23698            String::from_utf8_lossy(&out.stdout).trim().to_owned()
23699        };
23700        g(&["init", "-q", "-b", "main"]);
23701        g(&[
23702            "-c",
23703            "user.name=Test",
23704            "-c",
23705            "user.email=test@example.invalid",
23706            "commit",
23707            "-q",
23708            "--allow-empty",
23709            "-m",
23710            "feat: fixture",
23711        ]);
23712        let head = g(&["rev-parse", "HEAD"]);
23713        g(&["update-ref", "refs/remotes/origin/main", &head]);
23714        let evidence = crate::pr_outcome::summary_evidence(
23715            &runner.state,
23716            crate::pr_outcome::FinalHead::At(&head),
23717        );
23718        let generated = &runner.state.candidates[0].summary;
23719        runner.state.change_summary = Some(crate::run::ChangeSummary {
23720            base: head.clone(),
23721            head,
23722            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
23723            title: summary_title(generated).unwrap(),
23724            body: summary_without_title(generated),
23725        });
23726    }
23727
23728    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
23729        let mut runner = runner_at(RunStatus::Gating);
23730        runner.state = state_with_summary("add retries", JA_SUMMARY);
23731        if review_only {
23732            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
23733        }
23734        runner.state.repo = dir.to_owned();
23735        runner.state.candidates[0].worktree = dir.to_owned();
23736        runner.state.config.agents = agents;
23737        seed_public_summary(&mut runner);
23738        runner
23739    }
23740
23741    fn gate_questions(runner: &Runner) -> usize {
23742        ask::Questions::open()
23743            .list()
23744            .into_iter()
23745            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
23746            .count()
23747    }
23748
23749    #[tokio::test]
23750    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
23751        crate::run::pin_test_home();
23752        let dir = tempfile::tempdir().unwrap();
23753        let mut runner = rewrite_runner(
23754            dir.path(),
23755            true,
23756            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23757        );
23758        runner.state.config.roles.language_judge =
23759            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
23760        let winner = runner.state.candidates[0].clone();
23761        let msg = runner
23762            .guarded_pr_message(&winner, None, true)
23763            .await
23764            .unwrap()
23765            .expect("an English rewrite posts without asking");
23766        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
23767        assert!(msg.body.contains("<details>kept</details>"));
23768        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
23769        assert_eq!(gate_questions(&runner), 0);
23770        assert!(runner.state.github_text.is_none());
23771        assert!(!runner.state.seats.contains_key("impl-A"));
23772        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23773    }
23774
23775    #[tokio::test]
23776    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
23777        crate::run::pin_test_home();
23778        let dir = tempfile::tempdir().unwrap();
23779        let mut runner = rewrite_runner(
23780            dir.path(),
23781            false,
23782            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
23783        );
23784        runner.state.config.roles.language_judge =
23785            Some(crate::config::AgentChoice::One("beta".into()));
23786        runner.state.seats.insert(
23787            "impl-A".into(),
23788            SeatState::new("impl-A", "alpha", runner.state.seed),
23789        );
23790        let winner = runner.state.candidates[0].clone();
23791        let msg = runner
23792            .guarded_pr_message(&winner, None, true)
23793            .await
23794            .unwrap()
23795            .expect("rewritten");
23796        assert!(msg.body.contains("<details>kept</details>"));
23797        assert_eq!(runner.state.seats["impl-A"].turns, 0);
23798        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23799        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23800    }
23801
23802    #[tokio::test]
23803    async fn rewrite_reuses_a_resumable_implementer_session() {
23804        crate::run::pin_test_home();
23805        let dir = tempfile::tempdir().unwrap();
23806        let mut runner = rewrite_runner(
23807            dir.path(),
23808            false,
23809            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
23810        );
23811        runner.state.config.roles.language_judge =
23812            Some(crate::config::AgentChoice::One("beta".into()));
23813        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23814        seat.turns = 1;
23815        runner.state.seats.insert("impl-A".into(), seat);
23816        let winner = runner.state.candidates[0].clone();
23817        runner
23818            .guarded_pr_message(&winner, None, true)
23819            .await
23820            .unwrap()
23821            .expect("rewritten");
23822        assert_eq!(runner.state.seats["impl-A"].turns, 2);
23823        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
23824    }
23825
23826    #[tokio::test]
23827    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
23828        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
23829        // session, was out of quota, and nothing else was asked.
23830        crate::run::pin_test_home();
23831        let dir = tempfile::tempdir().unwrap();
23832        let mut exhausted = spec("alpha");
23833        exhausted.command = vec!["false".into()];
23834        let mut runner = rewrite_runner(
23835            dir.path(),
23836            false,
23837            vec![exhausted, scripted("beta", EN_REWRITE)],
23838        );
23839        // The implementer's own agent is in the role too: it is not asked twice.
23840        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23841            "alpha".into(),
23842            "beta".into(),
23843        ]));
23844        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
23845        seat.turns = 1;
23846        runner.state.seats.insert("impl-A".into(), seat);
23847        let winner = runner.state.candidates[0].clone();
23848        let msg = runner
23849            .guarded_pr_message(&winner, None, true)
23850            .await
23851            .unwrap()
23852            .expect("the role rewrites after the implementer fails");
23853        assert!(msg.body.contains("`src/net.rs`"));
23854        assert_eq!(
23855            runner.state.seats["impl-A"].turns, 2,
23856            "the implementer was tried once"
23857        );
23858        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
23859        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
23860    }
23861
23862    #[tokio::test]
23863    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
23864        crate::run::pin_test_home();
23865        let dir = tempfile::tempdir().unwrap();
23866        let mut broken = spec("bad");
23867        broken.command = vec!["false".into()];
23868        let mut runner = rewrite_runner(
23869            dir.path(),
23870            true,
23871            vec![
23872                broken,
23873                scripted("noformat", "Add retries."),
23874                scripted("good", EN_REWRITE),
23875            ],
23876        );
23877        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
23878            "bad".into(),
23879            "noformat".into(),
23880            "good".into(),
23881        ]));
23882        let winner = runner.state.candidates[0].clone();
23883        let msg = runner
23884            .guarded_pr_message(&winner, None, true)
23885            .await
23886            .unwrap()
23887            .expect("the third agent rewrites");
23888        assert!(msg.body.contains("`src/net.rs`"));
23889        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
23890        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
23891    }
23892
23893    #[tokio::test]
23894    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
23895        crate::run::pin_test_home();
23896        // (agents, role): still Japanese; no TITLE; nothing resolvable.
23897        for (agents, role, saved) in [
23898            (
23899                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
23900                Some("beta"),
23901                true,
23902            ),
23903            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
23904            (Vec::new(), None, false),
23905        ] {
23906            let dir = tempfile::tempdir().unwrap();
23907            let mut runner = rewrite_runner(dir.path(), true, agents);
23908            runner.state.config.roles.language_judge =
23909                role.map(|r| crate::config::AgentChoice::One(r.into()));
23910            let winner = runner.state.candidates[0].clone();
23911            let first = runner
23912                .guarded_pr_message(&winner, None, true)
23913                .await
23914                .unwrap();
23915            assert!(first.is_none(), "withheld text parks the run");
23916            assert!(runner.state.parked);
23917            assert_eq!(gate_questions(&runner), 1);
23918            let g = runner.state.github_text.as_ref().unwrap();
23919            assert_eq!(g.rewrite.is_some(), saved);
23920            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
23921        }
23922    }
23923
23924    #[tokio::test]
23925    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
23926        crate::run::pin_test_home();
23927        let dir = tempfile::tempdir().unwrap();
23928        let mut runner = runner_at(RunStatus::Gating);
23929        runner.state =
23930            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
23931        runner.state.repo = dir.path().to_owned();
23932        runner.state.candidates[0].worktree = dir.path().to_owned();
23933        let mut author = spec("alpha");
23934        author.command = vec![
23935            "sh".into(),
23936            "-c".into(),
23937            "printf '%s' \"$REWRITE_REPLY\"".into(),
23938        ];
23939        author.env.insert(
23940            "REWRITE_REPLY".into(),
23941            format!(
23942                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
23943                "English sentence. ".repeat(600)
23944            ),
23945        );
23946        runner.state.config.agents = vec![author];
23947        seed_public_summary(&mut runner);
23948        let winner = runner.state.candidates[0].clone();
23949        assert!(
23950            runner
23951                .guarded_pr_message(&winner, None, true)
23952                .await
23953                .unwrap()
23954                .is_none()
23955        );
23956        let record = runner.state.github_text.clone().unwrap();
23957        let rewrite = record.rewrite.clone().unwrap();
23958        assert_eq!(rewrite.failed, ["body-language"]);
23959        assert!(
23960            rewrite
23961                .body
23962                .ends_with(&format!("magi:run/{}", runner.state.id))
23963        );
23964        let store = ask::Questions::open();
23965        let q = store.get(record.question.as_ref().unwrap()).unwrap();
23966        assert!(
23967            q.choices
23968                .contains(&crate::github_text::USE_REWRITE.to_owned())
23969        );
23970        for label in [
23971            "ORIGINAL",
23972            "FALLBACK",
23973            "REWRITE",
23974            "Rules still failed: body-language",
23975            "truncated",
23976        ] {
23977            assert!(q.detail.contains(label), "{label}");
23978        }
23979        let artifact = std::fs::read_to_string(crate::run::artifact_path(
23980            &runner.state,
23981            &crate::github_text::artifact_name(&record.fingerprint),
23982        ))
23983        .unwrap();
23984        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
23985        // Restart from serialized state, then retry after a bad owner title.
23986        runner.state =
23987            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
23988        store
23989            .update(&q.id, |q| {
23990                q.say("まだ日本語")?;
23991                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
23992            })
23993            .unwrap();
23994        assert!(
23995            runner
23996                .guarded_pr_message(&winner, None, true)
23997                .await
23998                .unwrap()
23999                .is_none()
24000        );
24001        let retry = runner.state.github_text.as_ref().unwrap();
24002        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
24003        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
24004        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
24005        assert!(
24006            q.posting_comparison
24007                .as_ref()
24008                .unwrap()
24009                .matches_detail(&q.detail)
24010        );
24011        store
24012            .update(&q.id, |q| {
24013                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
24014            })
24015            .unwrap();
24016        let message = runner
24017            .guarded_pr_message(&winner, None, true)
24018            .await
24019            .unwrap()
24020            .unwrap();
24021        assert_eq!(message.title, rewrite.title);
24022        assert_eq!(message.body, rewrite.body);
24023        assert_eq!(
24024            runner.state.github_text.as_ref().unwrap().choice,
24025            Some(GithubTextChoice::Rewrite)
24026        );
24027        let again = runner
24028            .guarded_pr_message(&winner, None, true)
24029            .await
24030            .unwrap()
24031            .unwrap();
24032        assert_eq!(again.title, message.title);
24033        assert_eq!(again.body, message.body);
24034        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
24035        assert!(
24036            !runner
24037                .state
24038                .events
24039                .iter()
24040                .any(|e| e.message.contains(&rewrite.title)
24041                    || e.message.contains("English sentence."))
24042        );
24043    }
24044
24045    #[tokio::test]
24046    async fn historical_posting_reply_never_expires_into_fallback() {
24047        crate::run::pin_test_home();
24048        let dir = tempfile::tempdir().unwrap();
24049        let mut runner = runner_at(RunStatus::Gating);
24050        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
24051        runner.state.repo = dir.path().to_owned();
24052        runner.state.candidates[0].worktree = dir.path().to_owned();
24053        seed_public_summary(&mut runner);
24054        let winner = runner.state.candidates[0].clone();
24055        assert!(
24056            runner
24057                .guarded_pr_message(&winner, None, true)
24058                .await
24059                .unwrap()
24060                .is_none()
24061        );
24062        let id = runner
24063            .state
24064            .github_text
24065            .as_ref()
24066            .unwrap()
24067            .question
24068            .clone()
24069            .unwrap();
24070        let store = ask::Questions::open();
24071        let mut q = store.get(&id).unwrap();
24072        q.asked_at = "2020-01-01T00:00:00Z".parse().unwrap();
24073        q.answer(ask::Answer::Choice(
24074            crate::github_text::POST_ORIGINAL.into(),
24075        ))
24076        .unwrap();
24077        q.answer_delivered = true;
24078        store.put(&mut q).unwrap();
24079        let source = std::fs::read(store.path_of(&id)).unwrap();
24080        let migration = crate::run::home().join("notifications/task-chat-migration-v1");
24081        std::fs::create_dir_all(&migration).unwrap();
24082        std::fs::write(
24083            migration.join("state.json"),
24084            serde_json::to_vec(&serde_json::json!({
24085                "schema": 1, "sources": [id], "pending": [id]
24086            }))
24087            .unwrap(),
24088        )
24089        .unwrap();
24090        for _ in 0..2 {
24091            assert!(
24092                runner
24093                    .guarded_pr_message(&winner, None, true)
24094                    .await
24095                    .unwrap()
24096                    .is_none()
24097            );
24098            assert!(!runner.state.github_text.as_ref().unwrap().resolved);
24099            assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
24100        }
24101        let home = crate::run::home();
24102        crate::task_chat::sweep(
24103            &crate::queue::Queue::at(home.join("queue")),
24104            &crate::talk::Talks::at(home.join("talks")),
24105            &store,
24106            &home,
24107        );
24108        let outbox = crate::task_inquiry::Outbox::at(&home);
24109        outbox
24110            .update_decision(&id, |d| {
24111                d.accept(&format!("decision: {}", crate::github_text::POST_ORIGINAL))?;
24112                d.source_answer_at = q.answered_at;
24113                Ok(true)
24114            })
24115            .unwrap();
24116        outbox
24117            .settle(
24118                &id,
24119                "owner-chat",
24120                "Owner confirms no publication remains necessary",
24121                None,
24122            )
24123            .unwrap();
24124        assert!(
24125            runner
24126                .guarded_pr_message(&winner, None, true)
24127                .await
24128                .unwrap()
24129                .is_none()
24130        );
24131        assert!(!runner.state.github_text.as_ref().unwrap().resolved);
24132        assert_eq!(std::fs::read(store.path_of(&id)).unwrap(), source);
24133    }
24134
24135    #[tokio::test]
24136    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
24137        crate::run::pin_test_home();
24138        let dir = tempfile::tempdir().unwrap();
24139        let mut runner = runner_at(RunStatus::Gating);
24140        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
24141        runner.state.repo = dir.path().to_owned();
24142        runner.state.candidates[0].worktree = dir.path().to_owned();
24143        seed_public_summary(&mut runner);
24144        let winner = runner.state.candidates[0].clone();
24145        assert!(
24146            runner
24147                .guarded_pr_message(&winner, None, true)
24148                .await
24149                .unwrap()
24150                .is_none()
24151        );
24152        let record = runner.state.github_text.as_ref().unwrap();
24153        assert!(record.rewrite.is_none());
24154        // Records written before the additive field deserialize with no rewrite.
24155        let mut old = serde_json::to_value(record).unwrap();
24156        old.as_object_mut().unwrap().remove("rewrite");
24157        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
24158        assert!(old.rewrite.is_none());
24159        let id = record.question.clone().unwrap();
24160        ask::Questions::open()
24161            .update(&id, |q| {
24162                q.choices.push(crate::github_text::USE_REWRITE.into());
24163                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
24164            })
24165            .unwrap();
24166        let message = runner
24167            .guarded_pr_message(&winner, None, true)
24168            .await
24169            .unwrap()
24170            .unwrap();
24171        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
24172        assert_eq!(
24173            runner.state.github_text.as_ref().unwrap().choice,
24174            Some(GithubTextChoice::Fallback)
24175        );
24176    }
24177
24178    #[tokio::test]
24179    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
24180        crate::run::pin_test_home();
24181        let dir = tempfile::tempdir().unwrap();
24182        let mut runner = runner_at(RunStatus::Gating);
24183        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
24184        runner.state.repo = dir.path().to_owned();
24185        runner.state.candidates[0].worktree = dir.path().to_owned();
24186        seed_public_summary(&mut runner);
24187        let winner = runner.state.candidates[0].clone();
24188        assert!(
24189            runner
24190                .guarded_pr_message(&winner, None, true)
24191                .await
24192                .unwrap()
24193                .is_none()
24194        );
24195        let store = ask::Questions::open();
24196        let find = |run: &str| {
24197            store
24198                .list()
24199                .into_iter()
24200                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
24201                .unwrap()
24202        };
24203        let run = runner.state.id.clone();
24204        let q = find(&run);
24205        let fp = runner
24206            .state
24207            .github_text
24208            .as_ref()
24209            .unwrap()
24210            .fingerprint
24211            .clone();
24212        let name = crate::github_text::artifact_name(&fp);
24213        let path = crate::run::artifact_path(&runner.state, &name);
24214        assert!(q.detail.contains(&path.display().to_string()));
24215        assert!(q.detail.contains("title-language: title"));
24216        let saved = std::fs::read_to_string(&path).expect("artifact written");
24217        assert!(saved.contains("日本語のタイトル"));
24218        assert!(q.cwd.is_none());
24219        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
24220        // A bad replacement is refused and asked about once more.
24221        let say = |id: &str, text: &str, choice: &str| {
24222            store
24223                .update(id, |q| {
24224                    q.thread.push(ask::Turn {
24225                        who: ask::Who::Operator,
24226                        body: text.to_owned(),
24227                        at: jiff::Timestamp::now(),
24228                        note: None,
24229                    });
24230                    q.answer(ask::Answer::Choice(choice.to_owned()))
24231                })
24232                .unwrap();
24233        };
24234        say(&q.id, "まだ日本語", "use my text");
24235        assert!(
24236            runner
24237                .guarded_pr_message(&winner, None, true)
24238                .await
24239                .unwrap()
24240                .is_none()
24241        );
24242        let second = find(&run);
24243        assert_ne!(second.id, q.id);
24244        assert!(second.detail.contains("did not pass"));
24245        // The retry still compares the original text; the rejected
24246        // replacement is named by category only.
24247        assert!(second.detail.contains("日本語のタイトル"));
24248        assert!(!second.detail.contains("まだ日本語"));
24249        assert!(second.detail.contains(&path.display().to_string()));
24250        say(&second.id, "fix: retry failed requests", "use my text");
24251        let message = runner
24252            .guarded_pr_message(&winner, None, true)
24253            .await
24254            .unwrap()
24255            .expect("a vetted replacement is posted");
24256        assert_eq!(message.title, "fix: retry failed requests");
24257        assert_eq!(
24258            runner
24259                .state
24260                .github_text
24261                .as_ref()
24262                .unwrap()
24263                .chosen_title
24264                .as_deref(),
24265            Some("fix: retry failed requests")
24266        );
24267        assert!(runner.state.github_text.as_ref().unwrap().resolved);
24268        // Resumed again: the saved decision is applied, nothing is re-asked.
24269        let again = runner
24270            .guarded_pr_message(&winner, None, true)
24271            .await
24272            .unwrap()
24273            .unwrap();
24274        assert_eq!(again.title, message.title);
24275        assert_eq!(
24276            store
24277                .list()
24278                .into_iter()
24279                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
24280                .count(),
24281            2
24282        );
24283    }
24284
24285    #[test]
24286    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
24287        // What magi itself writes stays English under any configured language,
24288        // so a future localisation of these headings fails here. (The agents'
24289        // own text is also checked by the posting gate.)
24290        let mut state = state_with_summary(
24291            "add retries",
24292            "TITLE: fix(web): batch reads\n- reads run.json once",
24293        );
24294        state.config.graph.language = "ja".to_owned();
24295        let m = pr_message(&state, 'A');
24296        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
24297        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
24298
24299        // The task is the operator's own text: it goes in untouched, and the
24300        // fallback title (no summary) may be in its language too.
24301        let task = "今回やってほしいこと: results projector を直す";
24302        let mut state = state_with_summary(task, "- no title line");
24303        state.config.graph.language = "ja".to_owned();
24304        let m = pr_message(&state, 'A');
24305        assert_eq!(
24306            m.title,
24307            format!("chore: land candidate A of run {}", state.id)
24308        );
24309        assert!(
24310            m.body.contains(&format!(
24311                "<summary>Original task</summary>\n\n{task}\n\n</details>"
24312            )),
24313            "{}",
24314            m.body
24315        );
24316    }
24317
24318    #[test]
24319    fn pr_message_scrubs_home_paths_and_addresses() {
24320        let state = state_with_summary(
24321            "fix it in /Users/someone/src/x",
24322            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
24323        );
24324        let m = pr_message(&state, 'A');
24325        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
24326            assert!(!m.body.contains(leak), "{}", m.body);
24327        }
24328        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
24329    }
24330
24331    #[test]
24332    fn pr_message_survives_a_task_that_closes_details() {
24333        let state = state_with_summary("a </details> b", "TITLE: fix: x");
24334        let m = pr_message(&state, 'A');
24335        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
24336    }
24337
24338    #[test]
24339    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
24340        let cmd = manual_merge_command(
24341            MergeStyle::Squash,
24342            Path::new("/repo"),
24343            "b",
24344            "fix: \"quoted\" $(x) `y`\n\nbody",
24345        );
24346        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
24347    }
24348
24349    #[test]
24350    fn manual_merge_command_matches_the_configured_style() {
24351        let repo = Path::new("/repo");
24352        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
24353
24354        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
24355        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
24356
24357        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
24358        assert_eq!(
24359            squash,
24360            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
24361             \"Merge magi run 0832 (candidate A)\""
24362        );
24363
24364        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
24365        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
24366    }
24367
24368    #[test]
24369    fn a_nudge_gets_a_quarter_of_the_budget() {
24370        // The judge and implement budgets magi ships with.
24371        assert_eq!(retry_budget(secs(1200), true), secs(300));
24372        assert_eq!(retry_budget(secs(3600), true), secs(900));
24373    }
24374
24375    #[test]
24376    fn a_resent_prompt_keeps_the_whole_budget() {
24377        // The seat kept no context, so the retry is the original job again and
24378        // shortening it would only guarantee a second failure.
24379        assert_eq!(retry_budget(secs(1200), false), secs(1200));
24380        assert_eq!(retry_budget(secs(60), false), secs(60));
24381    }
24382
24383    #[test]
24384    fn the_floor_never_exceeds_the_original_budget() {
24385        // A short configured timeout must not be *raised* by the floor: the
24386        // operator asked for a bound, and a retry may not outlast the attempt
24387        // it is retrying.
24388        assert_eq!(retry_budget(secs(60), true), secs(60));
24389        assert_eq!(retry_budget(secs(480), true), secs(120));
24390        assert_eq!(retry_budget(secs(0), true), secs(0));
24391    }
24392
24393    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
24394        agent::CommandEvidence {
24395            id: "item1".to_owned(),
24396            description: "cargo test".to_owned(),
24397            exit_code,
24398            result_summary: String::new(),
24399            source: "codex".to_owned(),
24400        }
24401    }
24402
24403    #[test]
24404    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
24405        // No evidence is not the same fact as unconfirmed evidence: a
24406        // backend with no adapter, or a reply that ran no commands at all,
24407        // must not be misread as carrying a dangling job.
24408        assert!(!has_unconfirmed_command(&[]));
24409    }
24410
24411    #[test]
24412    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
24413        // Deliberately not a check on the exit code's *value*: a fixer
24414        // legitimately runs something that fails mid-iteration before it
24415        // succeeds, and that must never by itself reopen a valid report.
24416        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
24417        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
24418        assert!(!has_unconfirmed_command(&[
24419            evidence(Some(0)),
24420            evidence(Some(101))
24421        ]));
24422    }
24423
24424    #[test]
24425    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
24426        assert!(has_unconfirmed_command(&[
24427            evidence(Some(0)),
24428            evidence(None)
24429        ]));
24430    }
24431
24432    #[test]
24433    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
24434        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24435        assert_eq!(
24436            verified_noop_claim(true, &[], text).as_deref(),
24437            Some("already fixed by b32cfc4, on main.")
24438        );
24439    }
24440
24441    #[test]
24442    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
24443        // A timeout or a bad exit code reads as the ordinary loss it is,
24444        // whatever the reply's own prose claims.
24445        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24446        assert!(verified_noop_claim(false, &[], text).is_none());
24447    }
24448
24449    #[test]
24450    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
24451        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
24452        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
24453        // A confirmed command alongside the marker is fine.
24454        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
24455    }
24456
24457    #[test]
24458    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
24459        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
24460    }
24461
24462    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
24463    /// pair, in order, labelled A, B, C, ...
24464    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
24465        runner.state.candidates = shape
24466            .iter()
24467            .enumerate()
24468            .map(|(i, &(empty, verified))| Candidate {
24469                index: i,
24470                label: (b'A' + i as u8) as char,
24471                agent: "sonnet".to_owned(),
24472                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
24473                worktree: PathBuf::from(format!("/wt/{i}")),
24474                summary: String::new(),
24475                stat: String::new(),
24476                files: 0,
24477                commits: 0,
24478                empty,
24479                failed: None,
24480                verified_noop: verified.map(str::to_owned),
24481                duration_ms: 0,
24482                folded: false,
24483            })
24484            .collect();
24485    }
24486
24487    #[test]
24488    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
24489        ask_test_home();
24490        let mut runner = runner_at(RunStatus::Implementing);
24491        set_candidates(
24492            &mut runner,
24493            &[
24494                (true, Some("already on main at b32cfc4")),
24495                (true, Some("same fix, see the existing test")),
24496            ],
24497        );
24498
24499        runner
24500            .after_implement()
24501            .expect("a verified no-op is not an error");
24502
24503        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
24504    }
24505
24506    #[test]
24507    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
24508        ask_test_home();
24509        let mut runner = runner_at(RunStatus::Implementing);
24510        // Candidate A declares a verified no-op; candidate B simply wrote
24511        // nothing and said nothing about why. One candidate's claim is not
24512        // the whole run's agreement.
24513        set_candidates(
24514            &mut runner,
24515            &[(true, Some("already on main at b32cfc4")), (true, None)],
24516        );
24517
24518        let err = runner
24519            .after_implement()
24520            .expect_err("an unverified empty candidate must still fail the run");
24521
24522        assert!(
24523            err.to_string().contains("no candidate produced a change"),
24524            "{err}"
24525        );
24526        assert_eq!(runner.state.status, RunStatus::Failed);
24527    }
24528
24529    #[test]
24530    fn after_implement_still_fails_an_ordinary_all_empty_run() {
24531        ask_test_home();
24532        let mut runner = runner_at(RunStatus::Implementing);
24533        set_candidates(&mut runner, &[(true, None), (true, None)]);
24534
24535        let err = runner
24536            .after_implement()
24537            .expect_err("no candidate declared anything; this is an ordinary failure");
24538
24539        assert!(
24540            err.to_string().contains("no candidate produced a change"),
24541            "{err}"
24542        );
24543        assert_eq!(runner.state.status, RunStatus::Failed);
24544    }
24545}
24546
24547/// A graft rollback whose restore could not be confirmed (see
24548/// [`Runner::graft`]).
24549#[derive(Debug)]
24550struct GraftRestoreFailed(String);
24551
24552impl std::fmt::Display for GraftRestoreFailed {
24553    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24554        write!(
24555            f,
24556            "could not restore the winner to its pre-graft commit: {}",
24557            self.0
24558        )
24559    }
24560}
24561
24562impl std::error::Error for GraftRestoreFailed {}
24563
24564/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
24565/// not the agent: they never draw on an agent's own timeout, and running out of
24566/// them never reads as an agent failure.
24567#[cfg(test)]
24568mod resource_wait_tests {
24569    use super::*;
24570    use std::collections::BTreeMap;
24571
24572    fn sh(id: &str, script: &str) -> AgentSpec {
24573        AgentSpec {
24574            id: id.to_owned(),
24575            kind: crate::config::AgentKind::Command,
24576            model: None,
24577            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
24578            extra_args: Vec::new(),
24579            env: BTreeMap::new(),
24580            prompt_delivery: None,
24581            max_turn_secs: None,
24582        }
24583    }
24584
24585    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
24586        SeatJob {
24587            seat: SeatState::new("impl-A", &spec.id, 1),
24588            spec,
24589            cwd: dir.to_path_buf(),
24590            prompt: "do it".to_owned(),
24591            timeout,
24592            allow_write: true,
24593            sessions: false,
24594            artifacts: dir.join("art"),
24595            stem: "impl-A".to_owned(),
24596            handover: None,
24597        }
24598    }
24599
24600    fn state(resource_wait_secs: u64) -> RunState {
24601        let mut config = Config::default();
24602        config.graph.timeout_resource_wait = resource_wait_secs;
24603        RunState::new(
24604            PathBuf::from("/repo"),
24605            "main".to_owned(),
24606            "abc1234".to_owned(),
24607            "task".to_owned(),
24608            config,
24609        )
24610    }
24611
24612    fn hold(cache: &Path) -> crate::cache::Guard {
24613        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
24614        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
24615            crate::cache::AcquireOutcome::Acquired(g) => g,
24616            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
24617        }
24618    }
24619
24620    async fn run(
24621        jobs: Vec<SeatJob>,
24622        sem: Arc<Semaphore>,
24623        cache: Option<&Path>,
24624        st: &mut RunState,
24625    ) -> Vec<(usize, SeatState, AgentOutcome)> {
24626        let prompts = Prompts::default();
24627        let ctx = WaveCtx {
24628            run: "r",
24629            node: "implement",
24630            prompts: &prompts,
24631            cache,
24632            round: None,
24633            carry_seats: false,
24634            pauses: None,
24635        };
24636        wave(jobs, sem, &ctx, st, 0).await
24637    }
24638
24639    #[tokio::test]
24640    async fn isolated_editing_seat_wait_releases_only_after_its_own_target_is_free() {
24641        crate::run::pin_test_home();
24642        let home = crate::run::home();
24643        let tmp = tempfile::tempdir().unwrap();
24644        let repo = tmp.path().join("repo");
24645        let checkout = tmp.path().join("checkout");
24646        std::fs::create_dir(&repo).unwrap();
24647        git::git(&repo, &["init"]).await.unwrap();
24648        git::git(
24649            &repo,
24650            &[
24651                "-c",
24652                "user.name=Fixture",
24653                "-c",
24654                "user.email=fixture@example.invalid",
24655                "commit",
24656                "--allow-empty",
24657                "-m",
24658                "initial",
24659            ],
24660        )
24661        .await
24662        .unwrap();
24663        git::worktree_add_detached(&repo, &checkout, "HEAD")
24664            .await
24665            .unwrap();
24666        let target = crate::build_target::resolve(&home, &checkout).unwrap();
24667        let target_lease = hold(&target);
24668        let shared = tmp.path().join("unrelated");
24669        let _shared_lease = hold(&shared);
24670        let mut st = state(0);
24671        st.repo = repo.clone();
24672        st.config.verify.build_cache = crate::config::BuildCache::Isolated;
24673        st.config.verify.e2e = vec![format!(
24674            "CARGO_TARGET_DIR='{}' cargo test",
24675            git::path_arg(&shared)
24676        )];
24677        let config = tmp.path().join("config.toml");
24678        std::fs::write(&config, toml::to_string(&st.config).unwrap()).unwrap();
24679        let prompts = Prompts::default();
24680        let run_id = st.id.clone();
24681        let ctx = WaveCtx {
24682            run: &run_id,
24683            node: "implement",
24684            prompts: &prompts,
24685            cache: None,
24686            round: None,
24687            carry_seats: false,
24688            pauses: None,
24689        };
24690        let writer = job(sh("writer", "exit 99"), &checkout, Duration::from_secs(30));
24691        let results = wave(vec![writer], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0).await;
24692        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24693        assert_eq!(results[0].1.turns, 0);
24694        let saved = RunState::load_under(&run_id, &home).unwrap();
24695        let block = saved.resource_block.unwrap();
24696        assert!(matches!(
24697            block.cause,
24698            crate::run::ResourceBlockCause::IsolatedCacheTimeout { target: ref actual, .. }
24699                if actual == &target
24700        ));
24701        let queue = crate::queue::Queue::at(tmp.path().join("queue"));
24702        let mut task = crate::queue::Task::new(
24703            "build".into(),
24704            "build".into(),
24705            repo,
24706            crate::queue::Source::Human,
24707        );
24708        task.runs.push(run_id);
24709        task.hold_machine(Some(format!(
24710            "{}{}",
24711            crate::cache::RESOURCE_HOLD,
24712            block.description()
24713        )));
24714        queue.put(&mut task).unwrap();
24715        let questions = crate::ask::Questions::at(tmp.path().join("questions"));
24716        let triage =
24717            || crate::triage::run_once(&queue, &questions, Some(&config), Timestamp::now());
24718        let pending = triage();
24719        assert!(pending.resumed.is_empty() && pending.asked.is_empty());
24720        assert_eq!(
24721            queue.get(&task.id).unwrap().status,
24722            crate::queue::TaskStatus::Held
24723        );
24724        drop(target_lease);
24725        assert_eq!(triage().resumed, vec![task.id.clone()]);
24726        assert_eq!(
24727            queue.get(&task.id).unwrap().status,
24728            crate::queue::TaskStatus::Queued
24729        );
24730        assert!(crate::cache::in_use(&home, &shared));
24731    }
24732
24733    #[tokio::test]
24734    async fn verification_cache_wait_preserves_the_command_budget() {
24735        crate::run::pin_test_home();
24736        let tmp = tempfile::tempdir().unwrap();
24737        let cache = tmp.path().join("target");
24738        let guard = hold(&cache);
24739        let release = tokio::spawn(async move {
24740            tokio::time::sleep(Duration::from_millis(100)).await;
24741            guard.release();
24742        });
24743        let mut st = state(10);
24744        let budget = Duration::from_millis(50);
24745        let wait = repository_wait(&st, None);
24746        let (outcomes, _) = with_cache_lease(
24747            &mut st,
24748            Some(&cache),
24749            "gate",
24750            "gate",
24751            tmp.path(),
24752            "abc1234",
24753            budget,
24754            "test gate",
24755            &wait,
24756            |_, available| async move {
24757                assert_eq!(
24758                    available, budget,
24759                    "contention must not shorten verification"
24760                );
24761                (Vec::new(), false, Vec::new())
24762            },
24763        )
24764        .await
24765        .unwrap();
24766        release.await.unwrap();
24767        assert!(outcomes.is_empty(), "{outcomes:?}");
24768        assert!(
24769            st.events
24770                .iter()
24771                .any(|event| event.message.contains("waiting for"))
24772        );
24773        assert!(st.resource_wait.is_none());
24774        // The verification body no longer owns the lease after returning.
24775        hold(&cache).release();
24776    }
24777
24778    #[tokio::test]
24779    async fn verification_cache_wait_expires_without_starting_commands() {
24780        crate::run::pin_test_home();
24781        let tmp = tempfile::tempdir().unwrap();
24782        let cache = tmp.path().join("target");
24783        let _held = hold(&cache);
24784        let mut st = state(0);
24785        let wait = repository_wait(&st, None);
24786        let (outcomes, _) = with_cache_lease(
24787            &mut st,
24788            Some(&cache),
24789            "gate",
24790            "gate",
24791            tmp.path(),
24792            "abc1234",
24793            Duration::from_secs(60),
24794            "test gate",
24795            &wait,
24796            |_, _| async { panic!("commands must not start without the cache lease") },
24797        )
24798        .await
24799        .unwrap();
24800        assert_eq!(outcomes.len(), 1);
24801        assert!(outcomes[0].resource_blocked);
24802        assert!(st.resource_wait.is_none());
24803        assert!(matches!(
24804            st.resource_block.unwrap().cause,
24805            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
24806        ));
24807    }
24808
24809    #[tokio::test]
24810    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
24811        crate::run::pin_test_home();
24812        let tmp = tempfile::tempdir().unwrap();
24813        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
24814        let cache = tmp.path().join("cache");
24815        let mut state = state(21600);
24816        let wait = repository_wait(&state, None);
24817        let (outcomes, _) = with_cache_lease(
24818            &mut state,
24819            Some(&cache),
24820            "gate",
24821            "gate",
24822            tmp.path(),
24823            "head",
24824            Duration::from_secs(60),
24825            "gate",
24826            &wait,
24827            |_, _| async { panic!("freshness failure must not start verification") },
24828        )
24829        .await
24830        .unwrap();
24831        assert_eq!(outcomes.len(), 1);
24832        assert!(outcomes[0].resource_blocked);
24833        let block = state.resource_block.unwrap();
24834        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
24835        assert!(!block.description().contains("timed out"));
24836    }
24837
24838    #[tokio::test]
24839    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
24840        crate::run::pin_test_home();
24841        let tmp = tempfile::tempdir().unwrap();
24842        let cache = tmp.path().join("cache");
24843        let _held = hold(&cache);
24844        let mut state = state(21600);
24845        let pause = Pause::new();
24846        let interrupt = Pause::new();
24847        let cause = ParkCause::Operator;
24848        pause.park_for(cause.clone());
24849        let prompts = Prompts::default();
24850        let ctx = WaveCtx {
24851            run: "test",
24852            node: "fix",
24853            prompts: &prompts,
24854            cache: Some(&cache),
24855            round: None,
24856            carry_seats: false,
24857            pauses: Some((&pause, &interrupt)),
24858        };
24859        let result = wave(
24860            vec![job(
24861                sh("writer", "exit 99"),
24862                tmp.path(),
24863                Duration::from_secs(1),
24864            )],
24865            Arc::new(Semaphore::new(1)),
24866            &ctx,
24867            &mut state,
24868            0,
24869        )
24870        .await;
24871        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
24872        assert_eq!(result[0].1.turns, 0);
24873        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24874        assert_eq!(
24875            state.resource_block.unwrap().cause,
24876            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
24877        );
24878        assert!(state.active.is_empty());
24879    }
24880
24881    #[tokio::test]
24882    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
24883        crate::run::pin_test_home();
24884        let tmp = tempfile::tempdir().unwrap();
24885        let cache = tmp.path().join("target-a");
24886        let guard = hold(&cache);
24887        // Held for longer than the agent's whole 1s budget.
24888        tokio::spawn(async move {
24889            tokio::time::sleep(Duration::from_millis(1500)).await;
24890            guard.release();
24891        });
24892        let mut st = state(60);
24893        let results = run(
24894            vec![job(
24895                sh("alpha", "echo ok"),
24896                tmp.path(),
24897                Duration::from_secs(1),
24898            )],
24899            Arc::new(Semaphore::new(1)),
24900            Some(&cache),
24901            &mut st,
24902        )
24903        .await;
24904        assert!(
24905            matches!(results[0].2, AgentOutcome::Ok(_)),
24906            "the wait must not be charged to the agent"
24907        );
24908        assert!(st.resource_block.is_none());
24909        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
24910        assert!(st.resource_wait.is_none() && st.active.is_empty());
24911    }
24912
24913    #[tokio::test]
24914    #[cfg(unix)]
24915    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
24916        use std::os::fd::AsRawFd as _;
24917        crate::run::pin_test_home();
24918        let tmp = tempfile::tempdir().unwrap();
24919        crate::git::git(tmp.path(), &["init"]).await.unwrap();
24920        let common = tmp.path().join(".git");
24921        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
24922        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
24923        assert_eq!(
24924            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
24925            0
24926        );
24927        let mut st = state(1);
24928        st.config.coordination = crate::config::test_coordination();
24929        let mut reader = job(
24930            sh("reader", "echo must-not-run"),
24931            tmp.path(),
24932            Duration::from_secs(30),
24933        );
24934        reader.allow_write = false;
24935        let results = run(
24936            vec![reader.clone()],
24937            Arc::new(Semaphore::new(1)),
24938            None,
24939            &mut st,
24940        )
24941        .await;
24942        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
24943        assert_eq!(results[0].1.turns, 0);
24944        assert!(FailClass::of(&results[0].2).is_none());
24945        assert_eq!(
24946            st.resource_block.as_ref().unwrap().kind,
24947            "repository coordination"
24948        );
24949        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24950        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24951        drop(external);
24952        let editor = crate::repo_lock::SeatLease::for_checkout(
24953            tmp.path(),
24954            true,
24955            crate::config::test_coordination().as_ref(),
24956        )
24957        .await
24958        .unwrap();
24959        let pause = Pause::new();
24960        let interrupt = Pause::new();
24961        let prompts = Prompts::default();
24962        let ctx = WaveCtx {
24963            run: "r",
24964            node: "review",
24965            prompts: &prompts,
24966            cache: None,
24967            round: None,
24968            carry_seats: false,
24969            pauses: Some((&pause, &interrupt)),
24970        };
24971        let (results, ()) = tokio::join!(
24972            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
24973            async {
24974                tokio::time::sleep(Duration::from_millis(50)).await;
24975                pause.park();
24976            }
24977        );
24978        assert!(
24979            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
24980        );
24981        assert_eq!(results[0].1.turns, 0);
24982        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
24983        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
24984        drop(editor);
24985    }
24986
24987    #[tokio::test]
24988    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
24989        crate::run::pin_test_home();
24990        let tmp = tempfile::tempdir().unwrap();
24991        let cache = tmp.path().join("target-b");
24992        let _held = hold(&cache);
24993        let mut st = state(1);
24994        let results = run(
24995            vec![job(
24996                sh("alpha", "echo ok"),
24997                tmp.path(),
24998                Duration::from_secs(1),
24999            )],
25000            Arc::new(Semaphore::new(1)),
25001            Some(&cache),
25002            &mut st,
25003        )
25004        .await;
25005        let (_, seat, out) = &results[0];
25006        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
25007        assert!(wave_blocked(&results));
25008        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
25009        assert_eq!(seat.turns, 0, "nothing was asked");
25010        assert!(
25011            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
25012            "no agent was spawned"
25013        );
25014        assert!(FailClass::of(out).is_none(), "not a failure class");
25015        let block = st.resource_block.as_ref().expect("recorded for the resume");
25016        assert_eq!(block.kind, "shared build cache");
25017        assert!(matches!(
25018            block.cause,
25019            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
25020        ));
25021        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
25022        assert!(st.active.is_empty());
25023    }
25024
25025    #[tokio::test]
25026    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
25027        crate::run::pin_test_home();
25028        let tmp = tempfile::tempdir().unwrap();
25029        let mut st = state(1);
25030        let results = run(
25031            vec![job(
25032                sh("alpha", "echo ok"),
25033                tmp.path(),
25034                Duration::from_secs(1),
25035            )],
25036            Arc::new(Semaphore::new(0)),
25037            None,
25038            &mut st,
25039        )
25040        .await;
25041        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
25042        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
25043        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
25044    }
25045
25046    #[tokio::test]
25047    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
25048        crate::run::pin_test_home();
25049        let tmp = tempfile::tempdir().unwrap();
25050        let mut st = state(60);
25051        let results = run(
25052            vec![job(
25053                sh("alpha", "sleep 30"),
25054                tmp.path(),
25055                Duration::from_secs(1),
25056            )],
25057            Arc::new(Semaphore::new(1)),
25058            None,
25059            &mut st,
25060        )
25061        .await;
25062        let out = &results[0].2;
25063        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
25064        let class = FailClass::of(out).expect("a failure");
25065        assert_eq!(class, FailClass::Timeout);
25066        assert!(should_hand_over(None, &class), "existing handover policy");
25067        assert!(st.resource_block.is_none());
25068    }
25069}