Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::run::{
46    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
47    DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, GateFixRecord, JobRecord, JobStatus,
48    Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, QuotaLoss,
49    ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord, tail,
50    write_artifact,
51};
52use crate::verdict::{
53    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
54    ReviewVote, Severity,
55};
56
57/// How much verification output is kept and fed back to the fixer.
58const OUTPUT_TAIL: usize = 8_000;
59
60/// Bytes of a failing command's output kept in an event, so the reason a run
61/// stopped is readable from the report without opening `run.json`.
62const EVENT_OUTPUT_TAIL: usize = 2_000;
63
64/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
65/// command's pid before releasing the build cache's lease.
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
67
68/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
69/// command's pid to actually exit before giving up and releasing anyway.
70///
71/// A timeout means the process was asked to die (`kill_on_drop`,
72/// `start_kill`), not that it already has — on Windows in particular that can
73/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
74/// the instant the command returns would let the very next acquirer (this
75/// run's own next round, another run's verification, the janitor's prune)
76/// start touching the same directory while it might still be writing to it,
77/// so this polls the actual pid — real confirmation, not a fixed guess —
78/// until it is gone or this ceiling is reached. It is still not full
79/// process-tree reaping: a grandchild the timed-out process spawned and that
80/// outlives it independently is invisible to a pid check, and continuing to
81/// observe and collect *that* stays a different piece of work with its own
82/// owner. Set generously because the common case returns early the moment
83/// the pid is confirmed gone, not because every timeout pays this in full.
84const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
85
86/// Consecutive review rounds with no tree progress (see
87/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
88/// instead of spending the rest of the round budget.
89///
90/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
91/// legitimately finds nothing left to change (its previous round's fix already
92/// covered it, and this round's reviewers re-raised only nits) looks the same
93/// as one that is spinning, for exactly one round. Two in a row is where the
94/// two stop being distinguishable, and a review round on this workload has
95/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
96/// third attempt at a tree that has not moved twice running is pure cost.
97/// This does not touch `review_rounds` itself, which stays the operator's
98/// call.
99pub(crate) const STAGNANT_LIMIT: usize = 2;
100
101/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
102/// a base that moved before giving up and leaving the run `Blocked` for a
103/// person.
104///
105/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
106/// that keeps moving faster than a run can catch it is not something more
107/// rebasing fixes, it is a person's call. Not the same *number as*
108/// `land_rounds` - this budget is spent before a pull request exists, land's
109/// after - but bounded for the identical reason, so it uses the same
110/// default. Counted across both call sites in [`Runner::finish_after_tally`]
111/// (once before review, once before the gate), because either one finding
112/// the base still moving is the same signal.
113const BASE_SYNC_ROUNDS: usize = 4;
114
115/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
116/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
117/// reply held no [`FixReport`].
118///
119/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
120/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
121/// "I'll pause here until the `cargo make check` background run reports
122/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
123/// No `FixReport` was ever collected from that seat, and the run moved on to
124/// the next review round regardless.
125///
126/// Bounded independently of `review_rounds` and `graph.retries`: this
127/// recovers one seat's missing report mid-round, not a new round of review or
128/// an ordinary parse retry, and must not itself become the unbounded wait the
129/// rest of this module exists to avoid.
130const MAX_FIX_CONTINUATIONS: usize = 2;
131
132/// One queued agent invocation.
133///
134/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
135/// CLI hung up on its own stream is asked again from the same job rather than
136/// rebuilt from scratch. See [`Runner::resume_undelivered`].
137#[derive(Clone)]
138struct SeatJob {
139    spec: AgentSpec,
140    seat: SeatState,
141    cwd: PathBuf,
142    prompt: String,
143    timeout: Duration,
144    allow_write: bool,
145    sessions: bool,
146    artifacts: PathBuf,
147    stem: String,
148}
149
150/// How the graph reads one agent invocation.
151///
152/// Quota is split out from an ordinary failure on purpose: a rate-limited call
153/// is known to fail again if retried now, so the retry loop must not spend an
154/// attempt on it. `Dropped` is split out for the opposite reason: unlike
155/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
156/// error JSON, never the agent's answer — a caller that matched only
157/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
158/// left to read that JSON as if it were usable output. `resume_undelivered`
159/// is the only caller that acts on it; everywhere else it is reported like an
160/// ordinary failure.
161enum AgentOutcome {
162    /// A usable output.
163    Ok(AgentOutput),
164    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
165    Quota(AgentOutput),
166    /// The CLI hung up on its own stream after billed work. See
167    /// [`agent::AgentOutput::work_undelivered`].
168    Dropped(AgentOutput),
169    /// Any other failure: a timeout, a bad exit code, an empty reply.
170    Failed(String),
171}
172
173/// A request to park the run at its next node boundary.
174///
175/// Cloning is how the request travels: the loop keeps one handle and hands a
176/// clone to each [`Runner`], and every clone points at the same flag. There
177/// is no channel because there is nothing to send - the only message is
178/// "park", it is idempotent, and a flag cannot be missed by a receiver that
179/// was not listening yet.
180///
181/// The boundary is what makes this cheap. Every node writes the run's state
182/// before the next one starts, and every node skips what is already recorded:
183/// `prep` returns early once candidates exist, `implement` asks only the seats
184/// with nothing on disk, `judge` returns early once judgements exist. So a
185/// parked run resumes into exactly the node it stopped before, and no agent
186/// work is thrown away. Killing the process mid-node, by contrast, loses
187/// whatever the seats in flight had not yet written - which for an implement
188/// wave is an hour of paid work.
189///
190/// A [`Runner`] watches two independent handles of this type - see
191/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
192/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
193/// clone covering the whole daemon's lifetime and is never asked to un-park,
194/// which is correct exactly because nothing is dispatched after it fires.
195/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
196/// that parks for an interrupted task must go on to run other tasks
197/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
198/// reusing the daemon-wide one.
199#[derive(Debug, Clone, Default)]
200pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
201
202impl Pause {
203    /// A pause nobody has asked for yet.
204    #[must_use]
205    pub fn new() -> Self {
206        Self::default()
207    }
208
209    /// Ask the run to park at its next node boundary. Idempotent.
210    pub fn park(&self) {
211        self.0.store(true, Ordering::SeqCst);
212    }
213
214    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
215    /// fold into the run's own `park` event - so an operator reading the run
216    /// later knows this was a deliberate interrupt rather than a shutdown or
217    /// a binary swap. The first reason recorded wins; a park already in
218    /// flight is not relabelled by a second, unrelated request.
219    pub fn park_because(&self, reason: impl Into<String>) {
220        let mut reason_guard = self
221            .1
222            .lock()
223            .unwrap_or_else(std::sync::PoisonError::into_inner);
224        if reason_guard.is_none() {
225            *reason_guard = Some(reason.into());
226        }
227        drop(reason_guard);
228        self.park();
229    }
230
231    /// Has a park been asked for?
232    #[must_use]
233    pub fn parked(&self) -> bool {
234        self.0.load(Ordering::SeqCst)
235    }
236
237    /// Why the park was asked for, when the caller used [`Pause::park_because`].
238    #[must_use]
239    pub fn reason(&self) -> Option<String> {
240        self.1
241            .lock()
242            .unwrap_or_else(std::sync::PoisonError::into_inner)
243            .clone()
244    }
245}
246
247/// Drives one run.
248pub struct Runner {
249    /// Run state; public so the CLI can report on it.
250    pub state: RunState,
251    roles: ResolvedRoles,
252    sem: Arc<Semaphore>,
253    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
254    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
255    /// this is never the same handle as `interrupt`.
256    pause: Pause,
257    /// Set when `magi serve`'s interrupt scheduler wants this specific run
258    /// parked at its next node boundary, to let a task marked
259    /// [`crate::queue::Task::interrupt`] run alone before this one carries
260    /// on. Unlike `pause`, a fresh, unshared handle per run - see
261    /// [`Runner::watch_interrupt`].
262    interrupt: Pause,
263}
264
265/// The commit a run branches from: the base branch as the remote has it.
266///
267/// Two failures this replaces. A run used to branch off `HEAD` and so refused
268/// to start on a dirty tree, which made `magi serve` decline every task for as
269/// long as the operator had work in progress - most of the time. Branching off
270/// the *local* base branch fixed that and introduced a worse one: `land` merges
271/// the winner on GitHub, nothing updates the local ref, and the next run
272/// branches off a base missing everything the previous runs landed. Two tasks
273/// in a row from a phone would have had the second silently re-implementing
274/// against stale code and opening a pull request that reverted the first.
275///
276/// Only refs move here - no checkout, no local branch, no merge - so it is safe
277/// with uncommitted work in the tree. A machine with no network still starts:
278/// the fetch may fail and the local tip is used with a warning, because
279/// refusing to run offline is a worse failure than running against a base the
280/// operator can see for themselves.
281///
282/// One function, called by both entry points. Two answers to "where does a run
283/// branch from" is the kind of drift nobody notices until a diff is wrong.
284/// Bring the local `branch` in line with `<remote>/<branch>` before a review
285/// checks it out.
286///
287/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
288/// anything other than plain `git push` from this checkout (a jj colocated
289/// workspace, another clone) moves only the remote-tracking ref - so the local
290/// one can be a stale placeholder. It moves only when local is behind the remote or is an
291/// empty placeholder that diverged from it; unpushed local work is kept, and a real
292/// divergence is refused rather than guessed at.
293async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
294    let tracking = format!("{remote}/{branch}");
295    let fetched = git::fetch(repo, remote, branch).await;
296    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
297    let local_exists = git::branch_exists(repo, branch).await?;
298    if !fresh {
299        if !local_exists {
300            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
301        }
302        tracing::warn!(
303            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
304        );
305        return Ok(());
306    }
307    let remote_sha = git::rev_parse(repo, &tracking).await?;
308    if !local_exists {
309        git::git(repo, &["branch", branch, &tracking]).await?;
310        return Ok(());
311    }
312    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
313    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
314        return Ok(());
315    }
316    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
317        // Diverged. `reconcile` settles it only when it can prove nothing is
318        // lost: a local tip that is the remote's change rebased is pushed over
319        // it (lease pinned to the tip read here), a tip whose every extra
320        // commit is empty is a placeholder the remote's work replaced, and
321        // anything else is two different changes - a question for a person.
322        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
323            .await?
324        {
325            crate::reconcile::Reconciliation::Pushed => {
326                tracing::warn!(
327                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
328                    short(&local_sha),
329                    short(&remote_sha)
330                );
331                return Ok(());
332            }
333            crate::reconcile::Reconciliation::Placeholder => {}
334            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
335        }
336    }
337    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
338    if !out.ok() {
339        bail!(
340            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
341            short(&local_sha),
342            short(&remote_sha),
343            out.stderr
344        );
345    }
346    tracing::warn!(
347        "local `{branch}` was stale: fast-forwarded {} -> {}",
348        short(&local_sha),
349        short(&remote_sha)
350    );
351    Ok(())
352}
353
354async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
355    let tracking = format!("{remote}/{base_branch}");
356    let fetched = git::fetch(repo, remote, base_branch).await;
357    if let Ok(out) = &fetched
358        && out.ok()
359        && git::rev_exists(repo, &tracking).await
360    {
361        return git::rev_parse(repo, &tracking).await;
362    }
363    let why = match &fetched {
364        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
365        Ok(_) => format!("{remote} has no {base_branch}"),
366        Err(e) => e.to_string(),
367    };
368    tracing::warn!(
369        "could not read {tracking} ({why}); branching off the local \
370         {base_branch} instead, which may be behind"
371    );
372    git::rev_parse(repo, base_branch).await.with_context(|| {
373        format!(
374            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
375             branch that exists"
376        )
377    })
378}
379
380/// Exclusive claim on one run's `magi fix` step, released on drop — including
381/// on an early return or a panic.
382///
383/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
384/// manual `magi fix` invocations against the same run are otherwise
385/// invisible to each other and would race to remove and recreate the same
386/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
387/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
388/// which is only ever reclaimed later, out of band, by
389/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
390/// `magi fix` invocation is not necessarily running under either of those, so
391/// nothing would ever sweep a lock a killed or crashed process left behind.
392/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
393/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
394/// lease use: an unreadable or unparsable pid, or a liveness query the
395/// platform cannot answer, reads as alive and the lock is left in place.
396struct FixClaim {
397    path: PathBuf,
398}
399
400impl FixClaim {
401    fn acquire(dir: &Path) -> Result<Self> {
402        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
403        let path = dir.join("fix.lock");
404        match Self::create(&path) {
405            Ok(claim) => Ok(claim),
406            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
407                if Self::reclaim_if_dead(&path) {
408                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
409                } else {
410                    bail!(
411                        "another `magi fix` is already running for this run ({} exists)",
412                        path.display()
413                    )
414                }
415            }
416            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
417        }
418    }
419
420    fn create(path: &Path) -> std::io::Result<Self> {
421        let mut f = std::fs::OpenOptions::new()
422            .write(true)
423            .create_new(true)
424            .open(path)?;
425        use std::io::Write as _;
426        // Read back by `reclaim_if_dead` on a later, stuck invocation.
427        writeln!(f, "{}", std::process::id())?;
428        Ok(Self {
429            path: path.to_owned(),
430        })
431    }
432
433    /// True if the lock named a process confirmed dead, in which case it was
434    /// also removed. Never true on an unreadable file, an unparsable pid, or
435    /// a liveness query the platform cannot answer — see this type's own doc.
436    fn reclaim_if_dead(path: &Path) -> bool {
437        let dead = std::fs::read_to_string(path)
438            .ok()
439            .and_then(|body| body.trim().parse::<u32>().ok())
440            .is_some_and(|pid| !crate::proc::pid_alive(pid));
441        dead && std::fs::remove_file(path).is_ok()
442    }
443}
444
445impl Drop for FixClaim {
446    fn drop(&mut self) {
447        let _ = std::fs::remove_file(&self.path);
448    }
449}
450
451impl Runner {
452    /// Start a fresh run against `repo`.
453    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
454        let repo = git::toplevel(repo).await?;
455        let missing = agent::missing_programs(&config.agents);
456        if !missing.is_empty() {
457            bail!(
458                "these agent programs are not on PATH: {}. Fix the roster in \
459                 magi.toml or install them.",
460                missing.join(", ")
461            );
462        }
463        let base_branch = match config.merge.base.clone() {
464            Some(b) => b,
465            None => git::current_branch(&repo)
466                .await?
467                .context("HEAD is detached; set [merge] base in magi.toml")?,
468        };
469        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
470        // Still worth saying out loud. The operator's uncommitted work is not
471        // part of this run, and someone watching a candidate fail to use a
472        // change they just made deserves to know why.
473        if !git::is_clean(&repo).await? {
474            tracing::warn!(
475                "{} has uncommitted changes; they are not part of this run, \
476                 which branches off {base_branch} ({})",
477                repo.display(),
478                &base_commit[..base_commit.len().min(8)]
479            );
480        }
481        let roles = config.resolve_roles()?;
482        let max_parallel = config.graph.max_parallel.max(1);
483        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
484        state.event("start", format!("run {} created", state.id));
485        state.save()?;
486        Ok(Self {
487            state,
488            roles,
489            sem: Arc::new(Semaphore::new(max_parallel)),
490            pause: Pause::new(),
491            interrupt: Pause::new(),
492        })
493    }
494
495    /// Open a review-only run against work that already exists on `branch`.
496    ///
497    /// The expensive half of the graph is the implement wave — measured at
498    /// 111 and 134 internal tool-loop turns on this repository, against a
499    /// handful for a judge or a reviewer. The cheap half is worth running on
500    /// hand-written work too, and there was no way to reach it.
501    ///
502    /// No new state and no schema change are needed: a run with **one** viable
503    /// candidate and a tally already decided degrades `execute` to exactly
504    /// review → gate → merge, because `judge` skips a single-candidate field,
505    /// `deliberate` has fewer than two first choices to reconcile, `vote`
506    /// returns early, `tally` is already present and `fold_losers` has no
507    /// losers. Resuming such a run therefore does the right thing as well.
508    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
509        Self::review_taking_over(repo, branch, config, None).await
510    }
511
512    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
513    /// at the same task still has `branch` checked out, its worktree is
514    /// released first if that is safe (see [`crate::handover`]), and the
515    /// review refuses with the reason if it is not. `None` is a hand-run
516    /// review and behaves exactly as [`Runner::review`] always did.
517    pub async fn review_taking_over(
518        repo: &Path,
519        branch: &str,
520        config: Config,
521        takeover: Option<crate::handover::Takeover>,
522    ) -> Result<Self> {
523        let repo = git::toplevel(repo).await?;
524        let missing = agent::missing_programs(&config.agents);
525        if !missing.is_empty() {
526            bail!(
527                "these agent programs are not on PATH: {}. Fix the roster in \
528                 magi.toml or install them.",
529                missing.join(", ")
530            );
531        }
532        let base_branch = match config.merge.base.clone() {
533            Some(b) => b,
534            None => git::current_branch(&repo)
535                .await?
536                .context("HEAD is detached; set [merge] base in magi.toml")?,
537        };
538        if base_branch == branch {
539            bail!("`{branch}` is the base branch; there is nothing to review against");
540        }
541        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
542
543        let roles = config.resolve_roles()?;
544        let max_parallel = config.graph.max_parallel.max(1);
545        let mut state = RunState::new(
546            repo.clone(),
547            base_branch,
548            base_commit.clone(),
549            String::new(),
550            config,
551        );
552
553        // Released before anything else touches the branch: a stale local
554        // branch is moved with `git branch -f`, which git refuses while an
555        // earlier attempt's worktree still has it checked out. Everything
556        // after this point that can fail puts the old run back.
557        let released = match &takeover {
558            Some(takeover) => crate::handover::release(&repo, branch, &state.id, takeover).await?,
559            None => None,
560        };
561        if let Some(released) = &released {
562            state.event(
563                "release",
564                format!(
565                    "took `{branch}` over from run {}: its worktree was released",
566                    crate::run::short_of(&released.old_id)
567                ),
568            );
569        }
570        // The owner's answer to an earlier divergence question is applied
571        // here: after the release (git will not move a checked-out branch)
572        // and before the sync that would otherwise ask again.
573        if let Some(choice) = takeover.as_ref().and_then(|t| t.choice.as_ref())
574            && let Err(e) =
575                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
576                    .await
577        {
578            if let Some(released) = &released {
579                released.restore(&repo, branch).await;
580            }
581            return Err(e.context("applying the owner's answer about the diverged branch"));
582        }
583        let opened =
584            Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
585        if opened.is_err()
586            && let Some(released) = &released
587        {
588            released.restore(&repo, branch).await;
589        }
590        opened
591    }
592
593    /// The half of [`Runner::review_taking_over`] that can fail after an
594    /// earlier attempt's worktree was released.
595    async fn open_review(
596        repo: &Path,
597        branch: &str,
598        mut state: RunState,
599        roles: ResolvedRoles,
600        max_parallel: usize,
601        base_commit: String,
602    ) -> Result<Self> {
603        sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
604        // The commit subjects are the closest thing to a task statement that
605        // existing work carries, and the reviewers are told as much.
606        let log = git::log_oneline(repo, &base_commit, branch)
607            .await
608            .unwrap_or_default();
609        let instruction = format!(
610            "Review the work already on branch `{branch}`. There is no task \
611             statement: what the change claims to do is whatever its commits \
612             say.\n\n{}",
613            if log.trim().is_empty() {
614                "(no commit messages)"
615            } else {
616                log.trim()
617            }
618        );
619        state.instruction = instruction;
620
621        // An attached worktree, so the fixer's commits land on the branch under
622        // review rather than on a detached head nobody will look at again.
623        let worktree = state.worktree_root().join("under-review");
624        if let Some(parent) = worktree.parent() {
625            tokio::fs::create_dir_all(parent).await.ok();
626        }
627        let path = worktree.to_string_lossy().to_string();
628        git::git(repo, &["worktree", "add", &path, branch])
629            .await
630            .with_context(|| {
631                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
632            })?;
633
634        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
635            .await
636            .unwrap_or(0);
637        if commits == 0 {
638            git::worktree_remove(repo, &worktree).await.ok();
639            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
640        }
641        let files = git::changed_files(&worktree, &base_commit, "HEAD")
642            .await
643            .map(|f| f.len())
644            .unwrap_or(0);
645        if files == 0
646            && let (Ok(head_tree), Ok(base_tree)) = (
647                git::tree_of(&worktree, "HEAD").await,
648                git::tree_of(&worktree, &base_commit).await,
649            )
650            && head_tree == base_tree
651        {
652            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
653            git::worktree_remove(repo, &worktree).await.ok();
654            bail!(
655                "`{branch}` at {} has a tree identical to base {}; this usually means \
656                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
657                 against `{}/{branch}`) rather than an empty change",
658                short(&head),
659                short(&base_commit),
660                state.config.merge.remote
661            );
662        }
663        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
664            .await
665            .unwrap_or_default();
666
667        state.candidates.push(Candidate {
668            index: 0,
669            label: 'A',
670            // Not an agent id on purpose: nothing in the roster wrote this, and
671            // the stats tables must not credit anyone with a win for it.
672            agent: "(existing branch)".to_owned(),
673            branch: branch.to_owned(),
674            worktree,
675            summary: String::new(),
676            stat,
677            files,
678            commits,
679            empty: false,
680            failed: None,
681            verified_noop: None,
682            duration_ms: 0,
683            folded: false,
684        });
685        state.tally = Some(Tally {
686            first_choice: BTreeMap::from([('A', 0)]),
687            borda: BTreeMap::new(),
688            winner: 'A',
689            rankings: 0,
690            unanimous_initial: false,
691            deliberated: false,
692            changed_votes: 0,
693            unanimous_final: false,
694            tie_break: None,
695            // No panel sat, so no quorum applies. Zero judges is the correct
696            // number for work that never competed, and must not be reported as
697            // a collapsed panel.
698            judges: 0,
699            present: 0,
700            quorum: 0,
701            met_quorum: true,
702            uncontested: Some("review-only run: nothing competed".to_owned()),
703        });
704        state.status = RunStatus::Reviewing;
705        state.event(
706            "start",
707            format!(
708                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
709                state.id
710            ),
711        );
712        state.save()?;
713        Ok(Self {
714            state,
715            roles,
716            sem: Arc::new(Semaphore::new(max_parallel)),
717            pause: Pause::new(),
718            interrupt: Pause::new(),
719        })
720    }
721
722    /// Reopen an existing run.
723    pub fn resume(id: &str) -> Result<Self> {
724        let state = RunState::load(id)?;
725        if let Some(to) = &state.released_to {
726            bail!(
727                "run {} cannot be resumed: its worktree was released to run {}",
728                state.short(),
729                crate::run::short_of(to)
730            );
731        }
732        let roles = state.config.resolve_roles()?;
733        let max_parallel = state.config.graph.max_parallel.max(1);
734        Ok(Self {
735            state,
736            roles,
737            sem: Arc::new(Semaphore::new(max_parallel)),
738            pause: Pause::new(),
739            interrupt: Pause::new(),
740        })
741    }
742
743    /// Walk the graph to a terminal state, skipping nodes already recorded.
744    ///
745    /// Every way a run is driven - the queue loop, `magi run`, a resume from
746    /// the phone - ends here, so this is the one place a run that ended
747    /// Blocked / Stalled / Failed, or died with an error, is announced to the
748    /// notification centre. Best-effort: see [`crate::notices::raise`].
749    pub async fn execute(&mut self) -> Result<()> {
750        let result = self.execute_graph().await;
751        self.mark_driver_exited();
752        let ended = if result.is_err() {
753            Some(crate::notices::run_stopped(&self.state.id, &self.state))
754        } else {
755            crate::notices::run_ended(&self.state)
756        };
757        if let Some(notice) = ended {
758            crate::notices::raise(notice);
759        }
760        result
761    }
762
763    /// Record that this process no longer drives the run, so its pid (a
764    /// daemon's outlives the run) is not read as a live driver.
765    ///
766    /// Written onto the record as it is on disk, never this copy: another
767    /// process may have resumed the run (recording its own pid and clearing
768    /// the flag) or released its worktree since this copy was read, and
769    /// saving over that would mark a running driver dead. Only a record still
770    /// naming this process as the driver is touched.
771    fn mark_driver_exited(&mut self) {
772        self.state.driver_exited = true;
773        let pid = std::process::id();
774        let Ok(mut disk) = RunState::load(&self.state.id) else {
775            return;
776        };
777        if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
778            return;
779        }
780        disk.driver_exited = true;
781        if let Err(e) = disk.save() {
782            tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
783        }
784    }
785
786    async fn execute_graph(&mut self) -> Result<()> {
787        // Moving again, so it is no longer parked. Set before the walk rather
788        // than in `resume`, so every way of re-entering the graph clears it
789        // and a card cannot claim a run is waiting to be resumed while the
790        // agents are already working.
791        self.state.parked = false;
792        // Any seat this state still lists as answering belongs to whatever
793        // process last drove this run — this one included, if it crashed
794        // mid-wave. Cleared and flushed immediately, before anything else
795        // runs, so a resume can never show a seat as live when nothing is
796        // asking it anything yet; the node that actually dispatches the next
797        // wave repopulates it.
798        self.state.clear_active();
799        // Recorded in the same spot, and flushed together with the clear
800        // above: this is the pid a reader checks (`RunState::liveness`) when
801        // no daemon claim exists to answer "is a process still driving this
802        // run" — a plain `magi run` / `magi review` typed into a terminal
803        // claims nothing there. Always overwritten, never only-if-absent, so
804        // a resumed run's stale pid from a previous, possibly-dead process
805        // can never survive into this one's own report. Unlike
806        // `clear_active`, this changes on every single `execute()` call, so
807        // the save below is now unconditional rather than only-if-cleared.
808        //
809        // `driver_started_at` is recorded in the same breath, from this same
810        // pid, so `liveness` can tell a live pid that is genuinely still us
811        // apart from one the OS has since handed to an unrelated process —
812        // see that field's own doc for why the pid alone is not enough.
813        // A resume that raced a takeover: the record on disk says the worktree
814        // was handed to a later run after this copy was read. Saving over it
815        // would erase that and drive a run with nothing to run in.
816        if let Ok(disk) = RunState::load(&self.state.id)
817            && let Some(to) = &disk.released_to
818        {
819            bail!(
820                "run {} cannot continue: its worktree was released to run {}",
821                self.state.short(),
822                crate::run::short_of(to)
823            );
824        }
825        let pid = std::process::id();
826        self.state.driver_pid = Some(pid);
827        self.state.driver_started_at = crate::proc::process_started_at(pid);
828        self.state.driver_exited = false;
829        self.state.save()?;
830        // A run that already lost its quorum never resumes into the verdict
831        // machinery: `deliberate` and `vote` would otherwise clobber the
832        // stalled marker back to Voting and the run would keep going past a
833        // verdict that is no longer trustworthy. Everything already recorded is
834        // kept, so the run stays resumable (or foldable) for a human to pick up.
835        //
836        // On --resume the run gets one chance to repair itself: the seats a
837        // rate limit took out are re-asked. If their quota has since reset and
838        // the quorum is restored, the run picks up and finishes; otherwise it
839        // stays stale and still-resumable for a later retry. If it does not
840        // recover, the returned status stays `Stalled` and nothing was
841        // clobbered (the recovery only mutates entries for the lost seats).
842        if self.state.status == RunStatus::Stalled {
843            if self.recover_stall().await? {
844                self.finish_after_tally().await?;
845            } else {
846                // Still below quorum: persist the marker and stay resumable.
847                self.state.save()?;
848            }
849            return Ok(());
850        }
851        // A run parked inside `land` - watching CI, mid fix-round, or
852        // waiting on the owner's merge approval - resumes directly into it,
853        // never back through `prep`. Everything before `merge` already
854        // concluded; that is the only way `status` reaches `Landing` in the
855        // first place. Re-walking `review_loop` first would also be actively
856        // wrong: its own status recomputation (see its doc) treats any
857        // clean round as reason to set `status` to `Gating`, which would
858        // clobber this marker before `merge` ever ran, and this run would
859        // never find its way back into `land` at all.
860        if self.state.status == RunStatus::Landing {
861            self.run_land().await?;
862            // `run_land` may have settled the run right here - CI came back
863            // green and the PR merged, say - without ever passing back
864            // through `merge`'s own trailing call. Whatever it left `status`
865            // as is what this has to read.
866            self.settle_questions();
867            return Ok(());
868        }
869        self.prep().await?;
870        if self.park_here()? {
871            return Ok(());
872        }
873        self.advise().await?;
874        if self.park_here()? {
875            return Ok(());
876        }
877        self.implement().await?;
878        if self.park_here()? {
879            return Ok(());
880        }
881        // `after_implement` already saved the state and settled any open
882        // questions when it set this; nothing later in the graph has
883        // anything to judge.
884        if self.state.status == RunStatus::VerifiedNoop {
885            return Ok(());
886        }
887        self.judge().await?;
888        if self.park_here()? {
889            return Ok(());
890        }
891        self.deliberate().await?;
892        if self.park_here()? {
893            return Ok(());
894        }
895        self.vote().await?;
896        if self.park_here()? {
897            return Ok(());
898        }
899        self.tally()?;
900        // A verdict that lost its quorum is not trustworthy: do not review,
901        // gate, or merge on it. Everything already done is kept, so the run
902        // stays resumable (or foldable); the human can replace the agent that
903        // ran out of quota and pick it up.
904        if self.state.status == RunStatus::Stalled {
905            // Persist the stalled marker now — the normal end-of-execute save
906            // below is below this early return, and without it a resumed run
907            // would reload a pre-tally status and keep going.
908            self.state.save()?;
909            return Ok(());
910        }
911        self.finish_after_tally().await?;
912        Ok(())
913    }
914
915    /// Park here if asked to, recording it in the run's own timeline.
916    ///
917    /// Returns whether the caller should stop walking the graph. The state is
918    /// saved either way by the node that just finished; this adds the event so
919    /// the operator's card says why a run that is neither finished nor moving
920    /// is sitting where it is.
921    fn park_here(&mut self) -> Result<bool> {
922        // Either handle asking is enough - see `Pause`'s own doc for why
923        // they are never the same one. `interrupt` is checked second so a
924        // reason it carries is preferred in the message below over a plain
925        // shutdown park racing it at the same boundary.
926        if !self.pause.parked() && !self.interrupt.parked() {
927            return Ok(false);
928        }
929        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
930            Some(reason) => format!(
931                "parked after `{}` ({reason}) — resume to carry on from here",
932                self.state.status.as_str()
933            ),
934            None => format!(
935                "parked after `{}` — resume to carry on from here",
936                self.state.status.as_str()
937            ),
938        };
939        self.state.event("park", why);
940        self.state.parked = true;
941        self.state.save()?;
942        Ok(true)
943    }
944
945    /// Hand the runner the pause `magi serve`'s own shutdown watches.
946    pub fn on_pause(&mut self, pause: Pause) {
947        self.pause = pause;
948    }
949
950    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
951    /// scheduler asking this one run - and no other - to park so a task
952    /// marked [`crate::queue::Task::interrupt`] can run alone. See
953    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
954    /// handle.
955    pub fn watch_interrupt(&mut self, pause: Pause) {
956        self.interrupt = pause;
957    }
958
959    /// Abandon this run's own open questions, once `status` has actually
960    /// settled rather than merely paused.
961    ///
962    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
963    /// either back up with the candidates, the review round and the seat
964    /// sessions already on disk, so a question an implementer asked mid-round
965    /// may still get a real answer read by a real resume. Only the statuses
966    /// `resumable` excludes are actually final: the run merged, it reached
967    /// `Ready` with nothing left to do, it failed outright with no
968    /// established point to continue from, or every candidate agreed, with
969    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
970    /// every one of those the seat that asked is gone for good, exactly like
971    /// the run being deleted under `magi run rm` - so the same cleanup
972    /// applies, worded for what actually happened instead of "the run was
973    /// deleted".
974    ///
975    /// Best-effort and silent on success: called from every place `status`
976    /// can land on one of those three, including ones a resumed run revisits,
977    /// so it must cost nothing when there was nothing open to begin with.
978    fn settle_questions(&mut self) {
979        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
980            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
981        }
982    }
983
984    /// The tail of the graph after a trustworthy tally: fold losers, review,
985    /// gate, merge, and persist.
986    async fn finish_after_tally(&mut self) -> Result<()> {
987        self.fold_losers().await?;
988        // Before review starts, and again right before the gate: a run's
989        // review rounds can themselves take long enough for the base to move
990        // a second time, and the gate is the one node whose "green" gets
991        // acted on.
992        self.sync_to_base().await?;
993        self.review_loop().await?;
994        self.sync_to_base().await?;
995        self.gate().await?;
996        self.merge().await?;
997        self.state.save()?;
998        Ok(())
999    }
1000
1001    // ---------------------------------------------------------------- prep
1002
1003    async fn prep(&mut self) -> Result<()> {
1004        if !self.state.candidates.is_empty() {
1005            return Ok(());
1006        }
1007        self.state.status = RunStatus::Prep;
1008        let repo = self.state.repo.clone();
1009        let base = self.state.base_commit.clone();
1010        let root = self.state.worktree_root();
1011        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1012
1013        // The hook is the write-time half of the blindness contract; the
1014        // presentation filter in `blind` is the half that cannot be bypassed.
1015        let hooks_dir = self.state.dir().join("hooks");
1016        if self.state.config.blind.commit_msg_hook {
1017            std::fs::create_dir_all(&hooks_dir)
1018                .with_context(|| format!("create {}", hooks_dir.display()))?;
1019            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1020            let path = hooks_dir.join("commit-msg");
1021            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1022            make_executable(&path)?;
1023            // Ref-counted rather than a plain idempotent set: with more than
1024            // one run able to be in flight in the same repository at once
1025            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1026            // true?" check cannot tell "another run of mine still needs
1027            // this" from "nobody does", and the run that happens to finish
1028            // first would disable the hook out from under a sibling still
1029            // relying on it.
1030            git::acquire_worktree_config(&repo).await?;
1031            self.state.enabled_worktree_config = true;
1032        }
1033
1034        for (index, (spec, label)) in self
1035            .roles
1036            .implementers
1037            .clone()
1038            .into_iter()
1039            .zip(labels)
1040            .enumerate()
1041        {
1042            let branch = self.state.branch_for(label);
1043            let worktree = root.join(format!("cand-{label}"));
1044            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
1045            if self.state.config.blind.commit_msg_hook {
1046                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1047            }
1048            git::local_exclude(&worktree, "/.magi/").await?;
1049            self.state.candidates.push(Candidate {
1050                index,
1051                label,
1052                agent: spec.id.clone(),
1053                branch,
1054                worktree,
1055                summary: String::new(),
1056                stat: String::new(),
1057                files: 0,
1058                commits: 0,
1059                empty: false,
1060                failed: None,
1061                verified_noop: None,
1062                duration_ms: 0,
1063                folded: false,
1064            });
1065        }
1066
1067        for j in 1..=self.roles.judges.len() {
1068            let wt = root.join(format!("judge-{j}"));
1069            if !wt.exists() {
1070                git::worktree_add_detached(&repo, &wt, &base).await?;
1071            }
1072        }
1073
1074        // Disposable, detached checkouts for the design-deliberation stage's
1075        // advisor seats — the same shape as the judges' above, at the same
1076        // base commit, since advisors also only ever read. Sized off the
1077        // configured count directly rather than a resolved roster: unlike
1078        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1079        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1080        // `prep` has no `ResolvedRoles` field to read a count from here.
1081        if self.state.config.graph.advise {
1082            for k in 1..=self.state.config.graph.advisors {
1083                let wt = root.join(format!("advisor-{k}"));
1084                if !wt.exists() {
1085                    git::worktree_add_detached(&repo, &wt, &base).await?;
1086                }
1087            }
1088        }
1089
1090        // A judge cannot tell it is looking at its own patch — the seats keep
1091        // separate conversations — but a panel that shares agents with the
1092        // field is less independent than it looks, and that is worth saying out
1093        // loud once per run rather than leaving it in the config.
1094        let authors: Vec<&str> = self
1095            .roles
1096            .implementers
1097            .iter()
1098            .map(|a| a.id.as_str())
1099            .collect();
1100        let overlap: Vec<String> = self
1101            .roles
1102            .judges
1103            .iter()
1104            .enumerate()
1105            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1106            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1107            .collect();
1108        if !overlap.is_empty() {
1109            let note = format!(
1110                "{} also authored a candidate; blind, but the panel is less \
1111                 independent than {} distinct agents would be",
1112                overlap.join(", "),
1113                self.roles.judges.len()
1114            );
1115            self.state.event("prep", note);
1116        }
1117
1118        self.state.event(
1119            "prep",
1120            format!(
1121                "{} candidates, {} judges, base {} ({})",
1122                self.state.candidates.len(),
1123                self.roles.judges.len(),
1124                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1125                self.state.base_branch
1126            ),
1127        );
1128        self.state.status = RunStatus::Implementing;
1129        self.state.save()?;
1130        Ok(())
1131    }
1132
1133    // -------------------------------------------------------------- advise
1134
1135    /// The design-deliberation stage: independent, read-only advisor seats
1136    /// each sketch a design before any implementer touches the repository,
1137    /// and (when at least one produced a usable proposal) a synthesis seat
1138    /// blends them into a brief `implement` carries in every candidate's
1139    /// prompt.
1140    ///
1141    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1142    /// advisors` is the proposal count. Everything here is best-effort and
1143    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1144    /// that cannot reach quota, or a synthesis seat that produced nothing
1145    /// usable all leave `implement` exactly as it was before this stage
1146    /// existed — the task instruction alone — rather than failing the whole
1147    /// competition over an enrichment stage. Every outcome is still recorded
1148    /// as an event, so a run that got nothing from this stage says why.
1149    ///
1150    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1151    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1152    /// resumed run whose stage failed would re-run it, and re-spend the
1153    /// agent calls, on every reentry before `implement`.
1154    ///
1155    /// Also skipped once any candidate shows implementation progress — the
1156    /// exact predicate `implement` itself uses to decide a candidate is no
1157    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1158    /// is not enough: a run created by an older binary that predates this
1159    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1160    /// an already-`Implementing`-or-later run under this build would
1161    /// otherwise walk straight back through `prep` (a no-op once candidates
1162    /// exist) into this node and spawn every advisor seat against worktrees
1163    /// `prep` never recreated — after implementation has already started,
1164    /// which is exactly the invariant this stage exists to guarantee.
1165    async fn advise(&mut self) -> Result<()> {
1166        let implement_untouched = self
1167            .state
1168            .candidates
1169            .iter()
1170            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1171        if !self.state.config.graph.advise || self.state.advise_attempted {
1172            return Ok(());
1173        }
1174        if !implement_untouched {
1175            self.state.event(
1176                "advise",
1177                "skipping the design-deliberation stage: at least one \
1178                 candidate already shows implementation progress, so this \
1179                 run is past the point the stage exists to run before"
1180                    .to_owned(),
1181            );
1182            self.state.advise_attempted = true;
1183            self.state.save()?;
1184            return Ok(());
1185        }
1186        let run_id = self.state.id.clone();
1187        let prompts = self.state.config.prompts.clone();
1188        let instruction = self.state.instruction.clone();
1189        let language = self.state.config.graph.language.clone();
1190        let root = self.state.worktree_root();
1191        let n = self.state.config.graph.advisors;
1192        let where_recorded = self.state.dir().join("run.json");
1193
1194        let seats = match self.state.config.advisors() {
1195            Ok(seats) if !seats.is_empty() => seats,
1196            Ok(_) => {
1197                self.state.event(
1198                    "advise",
1199                    format!(
1200                        "[graph] advisors is 0; skipping the design-deliberation \
1201                         stage and continuing without a synthesis brief (see {})",
1202                        where_recorded.display()
1203                    ),
1204                );
1205                self.state.advise_attempted = true;
1206                self.state.save()?;
1207                return Ok(());
1208            }
1209            Err(e) => {
1210                self.state.event(
1211                    "advise",
1212                    format!(
1213                        "could not resolve advisor seats ({e:#}); continuing \
1214                         without a design-deliberation brief (see {})",
1215                        where_recorded.display()
1216                    ),
1217                );
1218                self.state.advise_attempted = true;
1219                self.state.save()?;
1220                return Ok(());
1221            }
1222        };
1223
1224        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1225        let artifacts = agent::artifacts_dir(&self.state.dir());
1226        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1227
1228        let mut jobs = Vec::new();
1229        for (i, spec) in seats.iter().cloned().enumerate() {
1230            let seat_key = format!("advisor-{}", i + 1);
1231            let seat = self.seat(&seat_key, &spec.id);
1232            jobs.push(SeatJob {
1233                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1234                spec,
1235                seat,
1236                cwd: worktrees[i % worktrees.len()].clone(),
1237                timeout,
1238                allow_write: false,
1239                sessions: false,
1240                artifacts: artifacts.clone(),
1241                stem: seat_key,
1242            });
1243        }
1244
1245        self.state.event(
1246            "advise",
1247            format!(
1248                "{} advisor seat(s) sketching a design in parallel",
1249                jobs.len()
1250            ),
1251        );
1252        let mut quota_losses = Vec::new();
1253        let cache = self.state.config.cache_dir();
1254        let ctx = WaveCtx {
1255            run: &run_id,
1256            node: "advise",
1257            prompts: &prompts,
1258            cache: cache.as_deref(),
1259            round: None,
1260        };
1261        let results = ask_json_wave::<Proposal>(
1262            jobs,
1263            Arc::clone(&self.sem),
1264            self.state.config.graph.retries,
1265            &ctx,
1266            &mut quota_losses,
1267            &mut self.state,
1268            &|p: &Proposal| p.validate(),
1269        )
1270        .await;
1271        self.state.quota.extend(quota_losses);
1272
1273        let mut records = Vec::with_capacity(results.len());
1274        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1275            let agent_id = seat.agent.clone();
1276            self.state.seats.insert(seat.key.clone(), seat);
1277            match res {
1278                Ok((proposal, out)) => {
1279                    self.state
1280                        .event("advise", format!("advisor-{} proposed a design", i + 1));
1281                    records.push(advise::AdvisorRecord::proposed(
1282                        i + 1,
1283                        agent_id,
1284                        proposal,
1285                        out.duration_ms,
1286                    ));
1287                }
1288                Err(e) => {
1289                    self.state.event(
1290                        "advise",
1291                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1292                    );
1293                    records.push(advise::AdvisorRecord::failed(
1294                        i + 1,
1295                        agent_id,
1296                        e.to_string(),
1297                    ));
1298                }
1299            }
1300        }
1301
1302        let mut advice = advise::Advice {
1303            records,
1304            synthesis: None,
1305        };
1306        if advice.proposals().is_empty() {
1307            self.state.event(
1308                "advise",
1309                "no advisor produced a usable proposal; continuing without a \
1310                 synthesis brief"
1311                    .to_owned(),
1312            );
1313        } else {
1314            match self
1315                .synthesize_brief(
1316                    &advice,
1317                    &instruction,
1318                    &language,
1319                    &worktrees[0],
1320                    &artifacts,
1321                    &run_id,
1322                    &prompts,
1323                    cache.as_deref(),
1324                )
1325                .await
1326            {
1327                Ok(Some(text)) => {
1328                    self.state.event(
1329                        "advise",
1330                        "synthesized a design brief for the implementer".to_owned(),
1331                    );
1332                    advice.synthesis = Some(text);
1333                }
1334                Ok(None) => {
1335                    self.state.event(
1336                        "advise",
1337                        "the synthesis seat produced nothing usable; continuing \
1338                         without a design brief"
1339                            .to_owned(),
1340                    );
1341                }
1342                Err(e) => {
1343                    self.state.event(
1344                        "advise",
1345                        format!("could not synthesize a design brief: {e:#}"),
1346                    );
1347                }
1348            }
1349        }
1350        advise::apply_reflection(&mut advice);
1351
1352        self.state.advice = Some(advice);
1353        self.state.advise_attempted = true;
1354        self.state.save()?;
1355        Ok(())
1356    }
1357
1358    /// The synthesis seat: reads every advisor's proposal and blends them
1359    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1360    /// out of [`Runner::advise`] only for readability — it is not called
1361    /// anywhere else.
1362    ///
1363    /// Picked the same way [`crate::talk`]'s standing conversation and
1364    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1365    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1366    /// order (a claude seat, else the first runnable agent in roster order)
1367    /// used when that field is unset — see `[roles] synthesizer`'s own doc
1368    /// in [`crate::config`] for why a dedicated field exists here at all.
1369    #[allow(clippy::too_many_arguments)]
1370    async fn synthesize_brief(
1371        &mut self,
1372        advice: &advise::Advice,
1373        instruction: &str,
1374        language: &str,
1375        cwd: &Path,
1376        artifacts: &Path,
1377        run_id: &str,
1378        prompts: &Prompts,
1379        cache: Option<&Path>,
1380    ) -> Result<Option<String>> {
1381        let want = self.state.config.roles.synthesizer.as_deref();
1382        let spec = agent::pick(&self.state.config.agents, want, &agent::installed)?;
1383        let mut seat = self.seat("advise-synthesis", &spec.id);
1384        let proposals = advice.proposals();
1385        let mut prompt = prompt::with_overlay(
1386            prompt::synthesize_brief(instruction, &proposals, language),
1387            prompts.overlay("advise"),
1388        );
1389        if cache.is_some() {
1390            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1391            // below — see `prompt::build_cache_note`'s doc for why telling a
1392            // read-only seat to build through the shared cache is exactly how
1393            // a sandbox's write refusal gets misread as a defect.
1394            prompt.push('\n');
1395            prompt.push_str(&prompt::build_cache_note("advise", false));
1396        }
1397        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1398        let out = agent::invoke(
1399            &spec,
1400            &mut seat,
1401            &Invocation {
1402                cwd,
1403                prompt: &prompt,
1404                timeout,
1405                allow_write: false,
1406                sessions: false,
1407                artifacts,
1408                stem: "advise-synthesis",
1409                run: run_id,
1410                node: "advise",
1411                cache_dir: None,
1412                attachments: &[],
1413            },
1414        )
1415        .await?;
1416        self.state.seats.insert(seat.key.clone(), seat);
1417        if !out.usable() {
1418            return Ok(None);
1419        }
1420        let text =
1421            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1422        Ok((!text.trim().is_empty()).then_some(text))
1423    }
1424
1425    // ----------------------------------------------------------- implement
1426
1427    async fn implement(&mut self) -> Result<()> {
1428        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1429        // agent files with `magi task add` name the run that paid for it. The
1430        // prompt overlay is cloned alongside it because the waves borrow it
1431        // while `self` is mutably borrowed by the node's own bookkeeping.
1432        let run_id = self.state.id.clone();
1433        let prompts = self.state.config.prompts.clone();
1434        let todo: Vec<usize> = self
1435            .state
1436            .candidates
1437            .iter()
1438            .enumerate()
1439            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1440            .map(|(i, _)| i)
1441            .collect();
1442        if todo.is_empty() {
1443            return self.after_implement();
1444        }
1445        self.state.status = RunStatus::Implementing;
1446
1447        let language = self.state.config.graph.language.clone();
1448        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1449        let sessions = self.state.config.graph.sessions;
1450        let artifacts = agent::artifacts_dir(&self.state.dir());
1451        // The design-deliberation stage's blended brief, when `advise` found
1452        // one — carried into every implementer's prompt the same way
1453        // regardless of which candidate it is.
1454        let brief = self
1455            .state
1456            .advice
1457            .as_ref()
1458            .and_then(|a| a.synthesis.as_deref())
1459            .map(str::to_owned);
1460
1461        let mut jobs = Vec::new();
1462        for &i in &todo {
1463            let (index, label, worktree) = {
1464                let c = &self.state.candidates[i];
1465                (c.index, c.label, c.worktree.clone())
1466            };
1467            let spec = self.roles.implementers[index].clone();
1468            let seat_key = format!("impl-{label}");
1469            let seat = self.seat(&seat_key, &spec.id);
1470            let instruction = self.state.instruction.clone();
1471            jobs.push(SeatJob {
1472                spec,
1473                seat,
1474                prompt: prompt::implement(
1475                    &instruction,
1476                    &worktree.to_string_lossy(),
1477                    &language,
1478                    brief.as_deref(),
1479                ),
1480                cwd: worktree,
1481                timeout,
1482                allow_write: true,
1483                sessions,
1484                artifacts: artifacts.clone(),
1485                stem: format!("impl-{label}"),
1486            });
1487        }
1488
1489        self.state.event(
1490            "implement",
1491            format!("{} candidates in parallel", jobs.len()),
1492        );
1493        // Kept so a seat whose CLI hung up can be asked again from the same
1494        // job: `wave` consumes what it is given. Mutable so `resume_quota_losses`
1495        // can update a seat's own entry once a fallback agent takes it over —
1496        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1497        // whichever agent actually answered, not the one that quota'd out.
1498        let mut sent = jobs.clone();
1499        let cache = self.state.config.cache_dir();
1500        let ctx = WaveCtx {
1501            run: &run_id,
1502            node: "implement",
1503            prompts: &prompts,
1504            cache: cache.as_deref(),
1505            round: None,
1506        };
1507        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1508        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1509            .await;
1510        self.resume_quota_losses(&mut results, &mut sent, &prompts, &run_id)
1511            .await;
1512        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1513            .await;
1514
1515        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1516            let seat_key = seat.key.clone();
1517            // A quota fallback (`resume_quota_losses`) may have handed this
1518            // seat to a different agent than the one `prep` recorded on the
1519            // candidate; the stats tables and any later fixer-defaults-to-
1520            // winner's-author lookup must credit whoever actually answered —
1521            // unless every fallback also quota'd out, in which case nobody
1522            // actually answered and crediting the last agent tried would
1523            // erase every earlier agent's own quota loss from the stats
1524            // tables instead of just this one seat's.
1525            let agent = seat.agent.clone();
1526            let exhausted_the_fallback_chain = matches!(&out, AgentOutcome::Quota(_));
1527            self.state.seats.insert(seat.key.clone(), seat);
1528            let label = self.state.candidates[i].label;
1529            let worktree = self.state.candidates[i].worktree.clone();
1530            let base = self.state.base_commit.clone();
1531
1532            let (summary, duration, failed, verified_claim) = match out {
1533                AgentOutcome::Ok(o) => {
1534                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1535                    let failed = (!o.usable()).then(|| {
1536                        if o.timed_out {
1537                            "agent timed out".to_owned()
1538                        } else {
1539                            format!("agent exited with {:?}", o.exit_code)
1540                        }
1541                    });
1542                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1543                    (text, o.duration_ms, failed, verified_claim)
1544                }
1545                // Left un-resumed by `resume_undelivered` (a dirty tree
1546                // already rescues the work, or there was no session left to
1547                // resume into) — reported like the ordinary failure it is,
1548                // never as if `o.text` (the CLI's raw error JSON) were an
1549                // answer.
1550                AgentOutcome::Dropped(o) => {
1551                    let why = o
1552                        .dropped
1553                        .as_ref()
1554                        .map(|d| d.why.as_str())
1555                        .unwrap_or("the CLI ended the stream without delivering its answer");
1556                    (
1557                        String::new(),
1558                        o.duration_ms,
1559                        Some(format!("the CLI dropped the stream ({why})")),
1560                        None,
1561                    )
1562                }
1563                AgentOutcome::Quota(o) => {
1564                    self.state.quota.push(QuotaLoss {
1565                        seat: seat_key,
1566                        node: "implement".to_owned(),
1567                        at: Timestamp::now(),
1568                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1569                    });
1570                    (
1571                        String::new(),
1572                        o.duration_ms,
1573                        Some("rate limited (quota); produced no change".to_owned()),
1574                        None,
1575                    )
1576                }
1577                AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1578            };
1579
1580            // Rescue anything the agent edited but never committed: an
1581            // uncommitted candidate would silently be an empty one.
1582            let rescued = match git::rescue_commit(
1583                &worktree,
1584                &format!("magi: candidate {label} (uncommitted work)"),
1585            )
1586            .await
1587            {
1588                Ok(r) => {
1589                    self.state.note_withheld("implement", &r.withheld);
1590                    r.committed
1591                }
1592                Err(_) => false,
1593            };
1594            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1595                .await
1596                .unwrap_or(0);
1597            let patch = git::diff(&worktree, &base, "HEAD")
1598                .await
1599                .unwrap_or_default();
1600            let stat = git::diff_stat(&worktree, &base, "HEAD")
1601                .await
1602                .unwrap_or_default();
1603            let files = git::changed_files(&worktree, &base, "HEAD")
1604                .await
1605                .map(|f| f.len())
1606                .unwrap_or(0);
1607            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1608
1609            let c = &mut self.state.candidates[i];
1610            if !exhausted_the_fallback_chain {
1611                c.agent = agent;
1612            }
1613            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1614            c.stat = stat;
1615            c.files = files;
1616            c.commits = commits;
1617            c.duration_ms = duration;
1618            c.empty = commits == 0 || patch.trim().is_empty();
1619            // An agent that failed but still produced a committed change stays
1620            // in the running: the patch is what gets judged, not the exit code.
1621            c.failed = match failed {
1622                Some(_) if c.empty => failed,
1623                _ => None,
1624            };
1625            // Only an empty candidate can be a verified no-op: a claim next
1626            // to a real patch is not what the marker is for, and `c.failed`
1627            // being `Some` here already implies `verified_claim` was never
1628            // set (see the guard above the match that produced it).
1629            c.verified_noop = if c.empty { verified_claim } else { None };
1630            let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1631                (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1632                (None, true, Some(_), _) => {
1633                    format!("candidate {label}: no change produced (agent-verified no-op)")
1634                }
1635                (None, true, None, _) => format!("candidate {label}: no change produced"),
1636                (None, false, _, true) => {
1637                    format!(
1638                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1639                    )
1640                }
1641                (None, false, _, false) => {
1642                    format!("candidate {label}: {files} files, {commits} commits")
1643                }
1644            };
1645            self.state.event("implement", note);
1646            self.state.save()?;
1647        }
1648
1649        self.after_implement()
1650    }
1651
1652    /// Ask again, once, for work a CLI did and then failed to hand over.
1653    ///
1654    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1655    /// status with an empty response and a usage report showing output tokens,
1656    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1657    /// seven minutes and 14,267 output tokens that arrived as an empty
1658    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1659    ///
1660    /// Two conditions, and both matter:
1661    ///
1662    /// - **Only when the tree is untouched.** Often the agent has already
1663    ///   written its files and only the closing message was lost; the rescue
1664    ///   commit below picks that up and there is nothing to ask for. Re-asking
1665    ///   then would pay for a second implementation of work already on disk.
1666    /// - **Once.** A CLI that drops one stream can drop the next, and this
1667    ///   node is the most expensive in the graph.
1668    ///
1669    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1670    /// dropped reply still carried its `conversation_id`, so the seat is asked
1671    /// to finish what it was doing rather than sent the whole task again. It
1672    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1673    /// node's - for the same reason a re-ranked judge does: restating finished
1674    /// work is not the work.
1675    ///
1676    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1677    /// same way until it resets, while an abandoned conversation is still
1678    /// there to be picked up.
1679    async fn resume_undelivered(
1680        &mut self,
1681        results: &mut [(usize, SeatState, AgentOutcome)],
1682        sent: &[SeatJob],
1683        prompts: &Prompts,
1684        run_id: &str,
1685    ) {
1686        for (wi, seat, out) in results.iter_mut() {
1687            let Some(dropped) = (match &*out {
1688                AgentOutcome::Dropped(o) => o.dropped.clone(),
1689                _ => None,
1690            }) else {
1691                continue;
1692            };
1693            let Some(job) = sent.get(*wi) else { continue };
1694            // Already on disk? Then only the closing message was lost.
1695            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1696                self.state.event(
1697                    "implement",
1698                    format!(
1699                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1700                         work is in the tree",
1701                        seat.key, dropped.output_tokens, dropped.why
1702                    ),
1703                );
1704                continue;
1705            }
1706            // The re-ask only makes sense as a resume: `resume_after_drop`
1707            // says nothing about the task, trusting the seat to still hold it.
1708            // Without a session to resume — sessions disabled, or this CLI's
1709            // drop shape happened not to carry a session id — that prompt
1710            // would open a brand-new conversation with no context at all,
1711            // which is worse than leaving this as the ordinary failure it
1712            // already is.
1713            if !has_context(&job.spec, seat, job.sessions) {
1714                self.state.event(
1715                    "implement",
1716                    format!(
1717                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1718                         is no session left to resume",
1719                        seat.key, dropped.output_tokens, dropped.why
1720                    ),
1721                );
1722                continue;
1723            }
1724            self.state.event(
1725                "implement",
1726                format!(
1727                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1728                     conversation",
1729                    seat.key, dropped.output_tokens, dropped.why
1730                ),
1731            );
1732            let mut retry = job.clone();
1733            retry.seat = seat.clone();
1734            retry.prompt = prompt::resume_after_drop(&dropped.why);
1735            retry.timeout = retry_budget(job.timeout, true);
1736            retry.stem = format!("{}-resume", job.stem);
1737            let cache = self.state.config.cache_dir();
1738            let ctx = WaveCtx {
1739                run: run_id,
1740                node: "implement",
1741                prompts,
1742                cache: cache.as_deref(),
1743                round: None,
1744            };
1745            let (resumed_seat, resumed) =
1746                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1747            *seat = resumed_seat;
1748            *out = resumed;
1749        }
1750    }
1751
1752    /// Fall an implement seat through to the next untried agent in the
1753    /// implementer roster when it lost to quota, instead of leaving the
1754    /// seat's loss final the moment one agent's account runs dry.
1755    ///
1756    /// Solo runs (`graph.candidates = 1`, `daemon::apply_solo`'s forced shape)
1757    /// are the motivating case: `Config::resolve_roles`'s `implementers`
1758    /// truncates to the single slot rotation picked, so a solo task whose one
1759    /// implementer hits quota mid-run used to have nothing else to try. This
1760    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
1761    /// unrotated roster — which is the only place the *other* candidates in
1762    /// the machine's roster still exist once `implementers` has been cut down
1763    /// to size.
1764    ///
1765    /// Walks forward from just past the seat's own original position in the
1766    /// roster, never wrapping back to the front: a later candidate slot (say
1767    /// `beta`, the roster's second entry) must fall through to the *next*
1768    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
1769    /// almost certainly a different candidate's own agent already — and once
1770    /// the roster's tail is exhausted there is nothing left to fall through
1771    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
1772    /// whole [`AgentSpec`]: a roster with the same id named twice must not
1773    /// let this retry that id forever. The loop keeps falling through until
1774    /// an attempt lands something other than `Quota` or the roster's tail
1775    /// runs out of untried ids, at which point the seat is left exactly as
1776    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
1777    /// `QuotaLoss` recorded, the candidate failed/empty.
1778    ///
1779    /// `sent` is taken mutably and updated with the fallback agent's spec:
1780    /// `resume_unconfirmed_commands`, which runs after this and also reads
1781    /// `sent`, must see whichever agent actually ended up answering the seat
1782    /// — reading the stale, original spec there would check session
1783    /// eligibility against the wrong CLI and could hand a fallback agent's
1784    /// session id to the agent that just lost the seat to quota.
1785    ///
1786    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
1787    /// — `self.seat` only reuses state when the agent id is unchanged, so
1788    /// handing it a different id already gets this for free. Reusing the old
1789    /// seat would resume a different CLI's session as if it were a
1790    /// continuation of this one.
1791    ///
1792    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
1793    /// a quota loss cuts an agent off mid-turn, so anything already in the
1794    /// tree is unfinished work, not a completed candidate a re-ask would pay
1795    /// for twice. A dirty tree is rescued into a commit first (the same
1796    /// neutral-identity rescue `implement`'s own outcome loop gives every
1797    /// candidate) so the next agent starts clean.
1798    ///
1799    /// The new agent gets the implementer's full prompt and full
1800    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
1801    /// it has no session and no context, and is implementing the task from
1802    /// nothing, unlike a resumed drop which is only restating work already
1803    /// done.
1804    ///
1805    /// Every intermediate `Quota` this loop absorbs is folded into a plain
1806    /// `implement` event, never into `self.state.quota` — that is what
1807    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
1808    /// go unspent, and a seat that ultimately recovered on its second or
1809    /// third agent is not the stalled panel that check exists to catch. Only
1810    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
1811    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
1812    /// outcome loop already has — this helper never pushes to it itself.
1813    async fn resume_quota_losses(
1814        &mut self,
1815        results: &mut [(usize, SeatState, AgentOutcome)],
1816        sent: &mut [SeatJob],
1817        prompts: &Prompts,
1818        run_id: &str,
1819    ) {
1820        let instruction = self.state.instruction.clone();
1821        let language = self.state.config.graph.language.clone();
1822        let brief = self
1823            .state
1824            .advice
1825            .as_ref()
1826            .and_then(|a| a.synthesis.as_deref())
1827            .map(str::to_owned);
1828        for (wi, seat, out) in results.iter_mut() {
1829            let Some(job) = sent.get_mut(*wi) else {
1830                continue;
1831            };
1832            // Where the seat's own original agent sits in the roster — the
1833            // fallback walk starts just past here, never at the front, so a
1834            // later candidate slot's quota loss does not fall back onto an
1835            // earlier slot's own agent.
1836            let start = self
1837                .roles
1838                .implementer_roster
1839                .iter()
1840                .position(|s| s.id == job.spec.id)
1841                .unwrap_or(0);
1842            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1843            let mut fallback_attempt = 0usize;
1844            while matches!(&*out, AgentOutcome::Quota(_)) {
1845                let Some(next) =
1846                    next_untried_implementer(&self.roles.implementer_roster, start, &tried)
1847                        .cloned()
1848                else {
1849                    break;
1850                };
1851                tried.insert(next.id.clone());
1852                fallback_attempt += 1;
1853
1854                if let Ok(r) = git::rescue_commit(
1855                    &job.cwd,
1856                    &format!(
1857                        "magi: candidate {} (uncommitted work before quota fallback)",
1858                        seat.key
1859                    ),
1860                )
1861                .await
1862                {
1863                    self.state.note_withheld("implement", &r.withheld);
1864                }
1865
1866                self.state.event(
1867                    "implement",
1868                    format!(
1869                        "{}: rate limited (quota) on {}; retrying with {}",
1870                        seat.key, seat.agent, next.id
1871                    ),
1872                );
1873
1874                let new_seat = self.seat(&seat.key, &next.id);
1875                // Kept in sync on `sent` itself, not just the local retry: a
1876                // later helper (`resume_unconfirmed_commands`) reads `sent`
1877                // after this one returns and must see whichever agent is now
1878                // occupying the seat, not the one that just quota'd out —
1879                // otherwise it would judge session/continuation eligibility
1880                // by the wrong CLI and could resend a fallback's session id
1881                // to the agent that lost it the seat in the first place.
1882                job.spec = next.clone();
1883                let mut retry = job.clone();
1884                retry.seat = new_seat;
1885                retry.prompt = prompt::implement(
1886                    &instruction,
1887                    &job.cwd.to_string_lossy(),
1888                    &language,
1889                    brief.as_deref(),
1890                );
1891                retry.stem = format!("{}-quota-{}", job.stem, next.id);
1892                let cache = self.state.config.cache_dir();
1893                let ctx = WaveCtx {
1894                    run: run_id,
1895                    node: "implement",
1896                    prompts,
1897                    cache: cache.as_deref(),
1898                    round: None,
1899                };
1900                let (fallback_seat, fallback_out) = run_one(
1901                    retry,
1902                    Arc::clone(&self.sem),
1903                    &ctx,
1904                    &mut self.state,
1905                    fallback_attempt,
1906                )
1907                .await;
1908                *seat = fallback_seat;
1909                *out = fallback_out;
1910            }
1911        }
1912    }
1913
1914    /// Ask an implement seat's own CLI to confirm what it started, once, when
1915    /// its reply reported a command whose completion status it never
1916    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1917    /// that does and does not mean.
1918    ///
1919    /// The completion contract this task asks for, extended to `implement`
1920    /// with the same signal `continue_fix_report` reads for the fixer,
1921    /// rather than a keyword search over the reply or a hard requirement on
1922    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1923    /// a candidate's CLI turn ended cleanly while a test run it had started
1924    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1925    /// named in it at all is untouched by this: `commands` is empty, so
1926    /// there is nothing to be unconfirmed.
1927    ///
1928    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1929    /// this is not about recovering edits that might already be on disk, it
1930    /// is about a result the seat itself never vouched for, which resuming
1931    /// asks for regardless of what the tree already holds. Bounded to one
1932    /// attempt for the same reason `resume_undelivered` is — this is the
1933    /// most expensive node in the graph — and a seat that still cannot
1934    /// confirm on that attempt is left as whatever its (possibly still
1935    /// unconfirmed) reply says; this does not invent a new "failed" reason
1936    /// for a candidate that otherwise produced a real, committed change.
1937    async fn resume_unconfirmed_commands(
1938        &mut self,
1939        results: &mut [(usize, SeatState, AgentOutcome)],
1940        sent: &[SeatJob],
1941        prompts: &Prompts,
1942        run_id: &str,
1943    ) {
1944        for (wi, seat, out) in results.iter_mut() {
1945            let AgentOutcome::Ok(o) = &*out else {
1946                continue;
1947            };
1948            if !has_unconfirmed_command(&o.commands) {
1949                continue;
1950            }
1951            let Some(job) = sent.get(*wi) else { continue };
1952            if !has_context(&job.spec, seat, job.sessions) {
1953                self.state.event(
1954                    "implement",
1955                    format!(
1956                        "{}: the reply named a command whose own CLI never confirmed the exit \
1957                         status of, but there is no session left to resume",
1958                        seat.key
1959                    ),
1960                );
1961                continue;
1962            }
1963            self.state.event(
1964                "implement",
1965                format!(
1966                    "{}: the reply named a command whose own CLI never confirmed the exit \
1967                     status of; resuming the conversation",
1968                    seat.key
1969                ),
1970            );
1971            let mut retry = job.clone();
1972            retry.seat = seat.clone();
1973            retry.prompt = prompt::resume_incomplete(
1974                "a command in your last reply had no confirmed exit status",
1975            );
1976            retry.timeout = retry_budget(job.timeout, true);
1977            retry.stem = format!("{}-confirm", job.stem);
1978            let cache = self.state.config.cache_dir();
1979            let ctx = WaveCtx {
1980                run: run_id,
1981                node: "implement",
1982                prompts,
1983                cache: cache.as_deref(),
1984                round: None,
1985            };
1986            let (resumed_seat, resumed) =
1987                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1988            *seat = resumed_seat;
1989            *out = resumed;
1990        }
1991    }
1992
1993    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1994    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1995    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1996    /// that motivated this.
1997    ///
1998    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1999    /// own nudge loop already covers for judge/review/vote seats, and not a
2000    /// dropped stream, which [`Runner::resume_undelivered`] covers for
2001    /// implement seats: here the CLI turn genuinely finished while the node's
2002    /// own work — the fixer's account of what it did — had not. Gated purely
2003    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
2004    /// reply, never on any wording in it, so a fixer whose valid, first-try
2005    /// `FixReport` happens to mention having waited on a background test is
2006    /// never resumed — the `Ok(report)` branch at the call site returns
2007    /// before this is ever invoked.
2008    ///
2009    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
2010    /// attempt ([`retry_budget`]), nothing attempted once the session is
2011    /// gone, and a quota hit ends the loop immediately rather than retrying a
2012    /// rate limit that fails the same way again.
2013    async fn continue_fix_report(
2014        &mut self,
2015        mut seat: SeatState,
2016        parse_err: String,
2017        job: &SeatJob,
2018        prompts: &Prompts,
2019        run_id: &str,
2020        round: usize,
2021    ) -> (
2022        SeatState,
2023        Option<FixReport>,
2024        Option<String>,
2025        ContinuationRecord,
2026    ) {
2027        let mut last_err = parse_err;
2028        let mut cumulative_wait_ms = 0u64;
2029        let mut attempts = 0usize;
2030        loop {
2031            if !has_context(&job.spec, &seat, job.sessions) {
2032                self.state.event(
2033                    "fix",
2034                    format!(
2035                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
2036                         session left to resume into"
2037                    ),
2038                );
2039                let outcome = if attempts == 0 {
2040                    ContinuationOutcome::NoSession
2041                } else {
2042                    ContinuationOutcome::Exhausted
2043                };
2044                return (
2045                    seat,
2046                    None,
2047                    Some(format!("unparsable fix report: {last_err}")),
2048                    ContinuationRecord {
2049                        attempts,
2050                        cumulative_wait_ms,
2051                        outcome,
2052                    },
2053                );
2054            }
2055            if attempts >= MAX_FIX_CONTINUATIONS {
2056                self.state.event(
2057                    "fix",
2058                    format!(
2059                        "round {round}: fixer's reply still had no adoption report after \
2060                         {attempts} continuation(s) ({last_err}); giving up"
2061                    ),
2062                );
2063                return (
2064                    seat,
2065                    None,
2066                    Some(format!(
2067                        "unparsable fix report after {attempts} continuation(s): {last_err}"
2068                    )),
2069                    ContinuationRecord {
2070                        attempts,
2071                        cumulative_wait_ms,
2072                        outcome: ContinuationOutcome::Exhausted,
2073                    },
2074                );
2075            }
2076            attempts += 1;
2077            self.state.event(
2078                "fix",
2079                format!(
2080                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2081                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2082                ),
2083            );
2084            let mut retry = job.clone();
2085            retry.seat = seat.clone();
2086            retry.prompt = prompt::resume_incomplete(&last_err);
2087            retry.timeout = retry_budget(job.timeout, true);
2088            retry.stem = format!("{}-continue{attempts}", job.stem);
2089            let cache = self.state.config.cache_dir();
2090            let ctx = WaveCtx {
2091                run: run_id,
2092                node: "fix",
2093                prompts,
2094                cache: cache.as_deref(),
2095                round: Some(round),
2096            };
2097            let (resumed_seat, resumed_out) = run_one(
2098                retry,
2099                Arc::clone(&self.sem),
2100                &ctx,
2101                &mut self.state,
2102                attempts,
2103            )
2104            .await;
2105            seat = resumed_seat;
2106            match resumed_out {
2107                AgentOutcome::Ok(o) => {
2108                    cumulative_wait_ms += o.duration_ms;
2109                    match verdict::extract_json::<FixReport>(&o.text) {
2110                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
2111                            self.state.event(
2112                                "fix",
2113                                format!(
2114                                    "round {round}: fixer's adoption report recovered after \
2115                                     {attempts} continuation(s)"
2116                                ),
2117                            );
2118                            return (
2119                                seat,
2120                                Some(report),
2121                                None,
2122                                ContinuationRecord {
2123                                    attempts,
2124                                    cumulative_wait_ms,
2125                                    outcome: ContinuationOutcome::Resumed,
2126                                },
2127                            );
2128                        }
2129                        // The report parsed, but this same reply's own
2130                        // CommandEvidence — the identical record `state.jobs`
2131                        // renders — names a command whose CLI never
2132                        // confirmed an exit status. Read together, that is
2133                        // not a resolved answer: keep nudging rather than
2134                        // accept a report standing next to a command the
2135                        // seat's own CLI cannot vouch for.
2136                        Ok(_) => {
2137                            last_err = "the reply parsed, but it reported a command whose own CLI \
2138                                 never confirmed an exit status"
2139                                .to_owned();
2140                        }
2141                        Err(e) => last_err = e.to_string(),
2142                    }
2143                }
2144                AgentOutcome::Quota(o) => {
2145                    cumulative_wait_ms += o.duration_ms;
2146                    self.state.quota.push(QuotaLoss {
2147                        seat: seat.key.clone(),
2148                        node: "fix".to_owned(),
2149                        at: Timestamp::now(),
2150                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2151                    });
2152                    self.state.event(
2153                        "fix",
2154                        format!(
2155                            "round {round}: continuation rate limited (quota); not retrying now"
2156                        ),
2157                    );
2158                    return (
2159                        seat,
2160                        None,
2161                        Some("rate limited (quota) while recovering the fix report".to_owned()),
2162                        ContinuationRecord {
2163                            attempts,
2164                            cumulative_wait_ms,
2165                            outcome: ContinuationOutcome::QuotaLost,
2166                        },
2167                    );
2168                }
2169                AgentOutcome::Dropped(o) => {
2170                    cumulative_wait_ms += o.duration_ms;
2171                    let why = o
2172                        .dropped
2173                        .as_ref()
2174                        .map(|d| d.why.as_str())
2175                        .unwrap_or("the CLI ended the stream without delivering its answer");
2176                    last_err = format!("the CLI dropped the stream ({why})");
2177                }
2178                AgentOutcome::Failed(e) => last_err = e,
2179            }
2180        }
2181    }
2182
2183    fn after_implement(&mut self) -> Result<()> {
2184        // Scan every candidate patch once the set is complete.
2185        if self.state.leaks.is_empty() {
2186            let cfg = self.state.config.blind.clone();
2187            let mut leaks = Vec::new();
2188            for c in &self.state.candidates {
2189                let Some(patch) =
2190                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2191                else {
2192                    continue;
2193                };
2194                leaks.extend(blind::scan(
2195                    &format!("candidate {} patch", c.label),
2196                    &patch,
2197                    &cfg.vendor_tokens,
2198                ));
2199            }
2200            if !leaks.is_empty() {
2201                let summary = leaks
2202                    .iter()
2203                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2204                    .collect::<Vec<_>>()
2205                    .join(", ");
2206                match cfg.on_leak {
2207                    LeakPolicy::Fail => {
2208                        self.state.status = RunStatus::Failed;
2209                        self.state
2210                            .event("blind", format!("vendor text in a patch: {summary}"));
2211                        self.state.leaks = leaks;
2212                        self.state.save()?;
2213                        self.settle_questions();
2214                        bail!(
2215                            "blind.on_leak = \"fail\" and vendor text reached a \
2216                             judged patch: {summary}"
2217                        );
2218                    }
2219                    LeakPolicy::Redact => self.state.event(
2220                        "blind",
2221                        format!("redacting vendor text for judging: {summary}"),
2222                    ),
2223                    LeakPolicy::Warn => self.state.event(
2224                        "blind",
2225                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
2226                    ),
2227                }
2228                self.state.leaks = leaks;
2229            }
2230        }
2231
2232        if self.state.viable().is_empty() {
2233            if self.state.all_candidates_verified_noop() {
2234                // Every candidate agreed, with evidence the adoption guard
2235                // accepted, that nothing belongs in this worktree. That is
2236                // not the same fact as a candidate that simply failed to
2237                // write anything, and settling it as an ordinary `Failed`
2238                // (see `SCHEMA`'s doc for schema 10) is what let two of
2239                // task 391f's attempts burn a retry each re-discovering the
2240                // same already-landed fix. Terminal either way, so `judge`
2241                // must never run over an empty candidate set — unlike the
2242                // `Failed` branch below this returns `Ok`, not an error:
2243                // nothing here failed.
2244                self.state.status = RunStatus::VerifiedNoop;
2245                self.state.save()?;
2246                self.settle_questions();
2247                return Ok(());
2248            }
2249            self.state.status = RunStatus::Failed;
2250            self.state.save()?;
2251            self.settle_questions();
2252            bail!("no candidate produced a change; nothing to judge");
2253        }
2254        self.state.status = RunStatus::Judging;
2255        self.state.save()?;
2256        Ok(())
2257    }
2258
2259    // --------------------------------------------------------------- judge
2260
2261    async fn judge(&mut self) -> Result<()> {
2262        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2263        // agent files with `magi task add` name the run that paid for it. The
2264        // prompt overlay is cloned alongside it because the waves borrow it
2265        // while `self` is mutably borrowed by the node's own bookkeeping.
2266        let run_id = self.state.id.clone();
2267        let prompts = self.state.config.prompts.clone();
2268        if !self.state.judgements.is_empty() || self.state.judge_skipped {
2269            return Ok(());
2270        }
2271        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2272        if viable.len() == 1 {
2273            // Recorded so this is a one-time event: `judgements` stays empty
2274            // either way, which without this flag is indistinguishable from
2275            // "not yet judged" on the next reentry — and status is left
2276            // untouched, so a later node's conclusion (e.g. `Blocked` after
2277            // the review budget ran out) survives a resume instead of being
2278            // clobbered back to `Judging` by this node running again.
2279            self.state.judge_skipped = true;
2280            self.state.event(
2281                "judge",
2282                format!(
2283                    "only candidate {} produced a change; judging skipped",
2284                    viable[0].label
2285                ),
2286            );
2287            self.state.save()?;
2288            return Ok(());
2289        }
2290        self.state.status = RunStatus::Judging;
2291
2292        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2293        let language = self.state.config.graph.language.clone();
2294        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2295        let sessions = self.state.config.graph.sessions;
2296        let artifacts = agent::artifacts_dir(&self.state.dir());
2297        let root = self.state.worktree_root();
2298        let base_short = short(&self.state.base_commit);
2299
2300        let mut jobs = Vec::new();
2301        let mut orders = Vec::new();
2302        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2303            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2304            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2305            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2306            let seat_key = format!("judge-{}", j + 1);
2307            let seat = self.seat(&seat_key, &spec.id);
2308            jobs.push(SeatJob {
2309                prompt: prompt::judge(
2310                    &self.state.instruction,
2311                    &views,
2312                    self.roles.judges.len(),
2313                    &base_short,
2314                    &language,
2315                ),
2316                spec,
2317                seat,
2318                cwd: root.join(format!("judge-{}", j + 1)),
2319                timeout,
2320                allow_write: false,
2321                sessions,
2322                artifacts: artifacts.clone(),
2323                stem: format!("judge-{}", j + 1),
2324            });
2325        }
2326
2327        self.state.event(
2328            "judge",
2329            format!(
2330                "{} judges ranking {} candidates blind",
2331                jobs.len(),
2332                viable.len()
2333            ),
2334        );
2335        let labels_for_check = labels.clone();
2336        let mut quota_losses = Vec::new();
2337        let cache = self.state.config.cache_dir();
2338        let ctx = WaveCtx {
2339            run: &run_id,
2340            node: "judge",
2341            prompts: &prompts,
2342            cache: cache.as_deref(),
2343            round: None,
2344        };
2345        let results = ask_json_wave::<Ranking>(
2346            jobs,
2347            Arc::clone(&self.sem),
2348            self.state.config.graph.retries,
2349            &ctx,
2350            &mut quota_losses,
2351            &mut self.state,
2352            &move |r: &Ranking| r.validate(&labels_for_check),
2353        )
2354        .await;
2355        self.state.quota.extend(quota_losses);
2356
2357        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2358            let agent_id = seat.agent.clone();
2359            self.state.seats.insert(seat.key.clone(), seat);
2360            let mut record = Judgement {
2361                judge: j + 1,
2362                seat: format!("judge-{}", j + 1),
2363                agent: agent_id,
2364                ranking: Vec::new(),
2365                reasons: BTreeMap::new(),
2366                confidence: None,
2367                order: orders[j].clone(),
2368                failed: None,
2369                duration_ms: 0,
2370            };
2371            match res {
2372                Ok((ranking, out)) => {
2373                    record.ranking = ranking.normalized();
2374                    record.reasons = ranking.reasons;
2375                    record.confidence = ranking.confidence;
2376                    record.duration_ms = out.duration_ms;
2377                    self.state.event(
2378                        "judge",
2379                        format!(
2380                            "judge {} ranked {}",
2381                            j + 1,
2382                            record.ranking.iter().collect::<String>()
2383                        ),
2384                    );
2385                }
2386                Err(e) => {
2387                    record.failed = Some(e.to_string());
2388                    self.state
2389                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2390                }
2391            }
2392            self.state.judgements.push(record);
2393            self.state.save()?;
2394        }
2395        Ok(())
2396    }
2397
2398    // ---------------------------------------------------------- deliberate
2399
2400    async fn deliberate(&mut self) -> Result<()> {
2401        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2402        // agent files with `magi task add` name the run that paid for it. The
2403        // prompt overlay is cloned alongside it because the waves borrow it
2404        // while `self` is mutably borrowed by the node's own bookkeeping.
2405        let run_id = self.state.id.clone();
2406        let prompts = self.state.config.prompts.clone();
2407        if !self.state.deliberation.is_empty() {
2408            return Ok(());
2409        }
2410        let tops: Vec<char> = self
2411            .state
2412            .judgements
2413            .iter()
2414            .filter_map(|j| j.ranking.first().copied())
2415            .collect();
2416        let rounds = self.state.config.graph.deliberate_rounds;
2417        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2418            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2419                self.state.event(
2420                    "deliberate",
2421                    format!("judges agreed on {} outright; no deliberation", tops[0]),
2422                );
2423            }
2424            self.state.status = RunStatus::Voting;
2425            self.state.save()?;
2426            return Ok(());
2427        }
2428
2429        self.state.status = RunStatus::Deliberating;
2430        self.state.event(
2431            "deliberate",
2432            format!(
2433                "split: first choices were {} — opening {rounds} round(s)",
2434                tops.iter().collect::<String>()
2435            ),
2436        );
2437
2438        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2439        let language = self.state.config.graph.language.clone();
2440        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2441        let sessions = self.state.config.graph.sessions;
2442        let artifacts = agent::artifacts_dir(&self.state.dir());
2443        let root = self.state.worktree_root();
2444        let base_short = short(&self.state.base_commit);
2445
2446        // Judges argue in sequence so that a turn can answer the one before it;
2447        // that is the difference between deliberation and three parallel
2448        // monologues.
2449        for round in 1..=rounds {
2450            let mut turns: Vec<DeliberationTurn> = Vec::new();
2451            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2452                if self.state.judgements[j].failed.is_some() {
2453                    continue;
2454                }
2455                let seat_key = format!("judge-{}", j + 1);
2456                let mut seat = self.seat(&seat_key, &spec.id);
2457                let transcript = self.transcript(&turns, j);
2458                let context = if has_context(&spec, &seat, sessions) {
2459                    None
2460                } else {
2461                    Some(self.candidate_block(&viable, &base_short))
2462                };
2463                let text = prompt::deliberate(
2464                    &self.state.instruction,
2465                    context.as_deref(),
2466                    &transcript,
2467                    round,
2468                    rounds,
2469                    &language,
2470                );
2471                let job = SeatJob {
2472                    spec,
2473                    seat: seat.clone(),
2474                    prompt: text,
2475                    cwd: root.join(format!("judge-{}", j + 1)),
2476                    timeout,
2477                    allow_write: false,
2478                    sessions,
2479                    artifacts: artifacts.clone(),
2480                    stem: format!("delib-{round}-judge-{}", j + 1),
2481                };
2482                let cache = self.state.config.cache_dir();
2483                let ctx = WaveCtx {
2484                    run: &run_id,
2485                    node: "deliberate",
2486                    prompts: &prompts,
2487                    cache: cache.as_deref(),
2488                    round: None,
2489                };
2490                let (updated, out) =
2491                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2492                seat = updated;
2493                let agent_id = seat.agent.clone();
2494                let seat_key = seat.key.clone();
2495                self.state.seats.insert(seat.key.clone(), seat);
2496                let body = match out {
2497                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2498                    // Never read the CLI's raw error JSON as this judge's
2499                    // position — skip the seat instead, the same as any other
2500                    // failed turn.
2501                    AgentOutcome::Dropped(o) => {
2502                        let why =
2503                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2504                                "the CLI ended the stream without delivering its answer",
2505                            );
2506                        self.state.event(
2507                            "deliberate",
2508                            format!(
2509                                "judge {} skipped: the CLI dropped the stream ({why})",
2510                                j + 1
2511                            ),
2512                        );
2513                        continue;
2514                    }
2515                    AgentOutcome::Quota(o) => {
2516                        self.state.quota.push(QuotaLoss {
2517                            seat: seat_key,
2518                            node: "deliberate".to_owned(),
2519                            at: Timestamp::now(),
2520                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2521                        });
2522                        self.state.event(
2523                            "deliberate",
2524                            format!("judge {} skipped: rate limited (quota)", j + 1),
2525                        );
2526                        continue;
2527                    }
2528                    AgentOutcome::Failed(e) => {
2529                        self.state
2530                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2531                        continue;
2532                    }
2533                };
2534                let tentative = verdict::extract_json::<Position>(&body)
2535                    .ok()
2536                    .and_then(|p| p.tentative)
2537                    .and_then(|s| s.trim().chars().next())
2538                    .map(|c| c.to_ascii_uppercase());
2539                self.state.event(
2540                    "deliberate",
2541                    format!(
2542                        "round {round}: judge {} now favours {}",
2543                        j + 1,
2544                        tentative.map_or("—".to_owned(), |c| c.to_string())
2545                    ),
2546                );
2547                turns.push(DeliberationTurn {
2548                    judge: j + 1,
2549                    agent: agent_id,
2550                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2551                    tentative,
2552                });
2553            }
2554            self.state
2555                .deliberation
2556                .push(DeliberationRound { round, turns });
2557            self.state.save()?;
2558        }
2559
2560        self.state.status = RunStatus::Voting;
2561        self.state.save()?;
2562        Ok(())
2563    }
2564
2565    // ---------------------------------------------------------------- vote
2566
2567    async fn vote(&mut self) -> Result<()> {
2568        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2569        // agent files with `magi task add` name the run that paid for it. The
2570        // prompt overlay is cloned alongside it because the waves borrow it
2571        // while `self` is mutably borrowed by the node's own bookkeeping.
2572        let run_id = self.state.id.clone();
2573        let prompts = self.state.config.prompts.clone();
2574        if !self.state.votes.is_empty() {
2575            return Ok(());
2576        }
2577        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2578        if viable.len() == 1 {
2579            return Ok(());
2580        }
2581        self.state.status = RunStatus::Voting;
2582
2583        let language = self.state.config.graph.language.clone();
2584        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2585        let sessions = self.state.config.graph.sessions;
2586        let artifacts = agent::artifacts_dir(&self.state.dir());
2587        let root = self.state.worktree_root();
2588        let base_short = short(&self.state.base_commit);
2589        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2590
2591        let mut jobs = Vec::new();
2592        let mut seats_at = Vec::new();
2593        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2594            if self
2595                .state
2596                .judgements
2597                .get(j)
2598                .is_some_and(|r| r.failed.is_some())
2599            {
2600                continue;
2601            }
2602            let seat_key = format!("judge-{}", j + 1);
2603            let seat = self.seat(&seat_key, &spec.id);
2604            let mut text = prompt::final_vote(&viable, &language);
2605            if !has_context(&spec, &seat, sessions) {
2606                text = format!(
2607                    "{}\n\n# Candidates\n\n{}",
2608                    text,
2609                    self.candidate_block(&candidates, &base_short)
2610                );
2611            }
2612            jobs.push(SeatJob {
2613                spec,
2614                seat,
2615                prompt: text,
2616                cwd: root.join(format!("judge-{}", j + 1)),
2617                timeout,
2618                allow_write: false,
2619                sessions,
2620                artifacts: artifacts.clone(),
2621                stem: format!("vote-judge-{}", j + 1),
2622            });
2623            seats_at.push(j);
2624        }
2625
2626        self.state.event(
2627            "vote",
2628            format!(
2629                "collecting {} final votes one by one, privately",
2630                jobs.len()
2631            ),
2632        );
2633        let allowed = viable.clone();
2634        let mut quota_losses = Vec::new();
2635        let cache = self.state.config.cache_dir();
2636        let ctx = WaveCtx {
2637            run: &run_id,
2638            node: "vote",
2639            prompts: &prompts,
2640            cache: cache.as_deref(),
2641            round: None,
2642        };
2643        let results = ask_json_wave::<FinalVote>(
2644            jobs,
2645            Arc::clone(&self.sem),
2646            self.state.config.graph.retries,
2647            &ctx,
2648            &mut quota_losses,
2649            &mut self.state,
2650            &move |v: &FinalVote| match v.label() {
2651                Some(c) if allowed.contains(&c) => Ok(()),
2652                other => bail!("vote {other:?} is not one of {allowed:?}"),
2653            },
2654        )
2655        .await;
2656        self.state.quota.extend(quota_losses);
2657
2658        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2659            let agent_id = seat.agent.clone();
2660            self.state.seats.insert(seat.key.clone(), seat);
2661            let initial = self
2662                .state
2663                .judgements
2664                .get(j)
2665                .and_then(|r| r.ranking.first().copied());
2666            let mut record = VoteRecord {
2667                judge: j + 1,
2668                agent: agent_id,
2669                vote: None,
2670                reason: String::new(),
2671                changed: false,
2672            };
2673            match res {
2674                Ok((v, _)) => {
2675                    record.vote = v.label();
2676                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2677                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2678                    self.state.event(
2679                        "vote",
2680                        format!(
2681                            "judge {} voted {}{}",
2682                            j + 1,
2683                            record.vote.unwrap_or('?'),
2684                            if record.changed { " (changed)" } else { "" }
2685                        ),
2686                    );
2687                }
2688                Err(e) => {
2689                    self.state
2690                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2691                }
2692            }
2693            self.state.votes.push(record);
2694            self.state.save()?;
2695        }
2696        Ok(())
2697    }
2698
2699    // --------------------------------------------------------------- tally
2700
2701    fn tally(&mut self) -> Result<()> {
2702        if self.state.tally.is_some() {
2703            return Ok(());
2704        }
2705        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2706        let tops: Vec<char> = self
2707            .state
2708            .judgements
2709            .iter()
2710            .filter_map(|j| j.ranking.first().copied())
2711            .collect();
2712        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2713
2714        // A judge whose private vote failed still counted once, in the initial
2715        // ranking; using it beats discarding a whole seat.
2716        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2717        let mut cast: Vec<char> = Vec::new();
2718        for (i, j) in self.state.judgements.iter().enumerate() {
2719            let vote = self
2720                .state
2721                .votes
2722                .iter()
2723                .find(|v| v.judge == i + 1)
2724                .and_then(|v| v.vote)
2725                .or_else(|| j.ranking.first().copied());
2726            if let Some(v) = vote {
2727                *first_choice.entry(v).or_insert(0) += 1;
2728                cast.push(v);
2729            }
2730        }
2731
2732        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2733        for j in &self.state.judgements {
2734            let n = j.ranking.len();
2735            for (pos, label) in j.ranking.iter().enumerate() {
2736                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2737            }
2738        }
2739
2740        let best = first_choice.values().copied().max().unwrap_or(0);
2741        let mut leaders: Vec<char> = first_choice
2742            .iter()
2743            .filter(|(_, v)| **v == best)
2744            .map(|(k, _)| *k)
2745            .collect();
2746        let mut tie_break = None;
2747        if leaders.len() > 1 {
2748            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2749            let borda_leaders: Vec<char> = leaders
2750                .iter()
2751                .copied()
2752                .filter(|l| borda[l] == top_borda)
2753                .collect();
2754            tie_break = Some(if borda_leaders.len() == 1 {
2755                format!(
2756                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2757                    leaders.len()
2758                )
2759            } else {
2760                format!(
2761                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2762                    leaders.len()
2763                )
2764            });
2765            leaders = borda_leaders;
2766            leaders.sort_unstable();
2767        }
2768        let winner = *leaders
2769            .first()
2770            .or(viable.first())
2771            .context("no candidate to declare a winner from")?;
2772
2773        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2774        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2775        let deliberated = !self.state.deliberation.is_empty();
2776
2777        // Whose verdict is this? A rate-limited seat is absent even if it
2778        // ranked before the limit hit, so presence is measured against the
2779        // recorded losses, not just "did a ranking ever appear".
2780        let quota_seats: std::collections::BTreeSet<&str> =
2781            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2782        let mut present = 0usize;
2783        for (i, j) in self.state.judgements.iter().enumerate() {
2784            if quota_seats.contains(j.seat.as_str()) {
2785                continue;
2786            }
2787            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2788            let voted = self
2789                .state
2790                .votes
2791                .iter()
2792                .any(|v| v.judge == i + 1 && v.vote.is_some());
2793            if ranked || voted {
2794                present += 1;
2795            }
2796        }
2797        // Strict majority of the configured panel. A bare majority is real
2798        // signal we can act on, while a minority verdict must never stand in
2799        // for a healthy one. A one-candidate run needs no panel at all, and
2800        // `judges` stays `0` rather than the roster size a panel that never
2801        // sat would otherwise be credited with.
2802        let needs_quorum = viable.len() > 1;
2803        let judges_total = if needs_quorum {
2804            self.roles.judges.len()
2805        } else {
2806            0
2807        };
2808        let quorum = if needs_quorum {
2809            judges_total / 2 + 1
2810        } else {
2811            0
2812        };
2813        let met_quorum = !needs_quorum || present >= quorum;
2814        let uncontested = (!needs_quorum).then(|| {
2815            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2816        });
2817
2818        self.state.event(
2819            "tally",
2820            match &uncontested {
2821                Some(reason) => format!("winner {winner} — {reason}"),
2822                None => format!(
2823                    "winner {winner} — votes {} | initial {} | {} changed | \
2824                     {present}/{judges_total} judges{}",
2825                    first_choice
2826                        .iter()
2827                        .map(|(k, v)| format!("{k}:{v}"))
2828                        .collect::<Vec<_>>()
2829                        .join(" "),
2830                    if unanimous_initial {
2831                        "unanimous"
2832                    } else {
2833                        "split"
2834                    },
2835                    changed_votes,
2836                    if met_quorum {
2837                        String::new()
2838                    } else {
2839                        format!(" — below quorum ({quorum} required)")
2840                    },
2841                ),
2842            },
2843        );
2844        if !met_quorum {
2845            self.state.event(
2846                "stall",
2847                format!(
2848                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2849                     the run stops here, resumable"
2850                ),
2851            );
2852        }
2853        self.state.tally = Some(Tally {
2854            first_choice,
2855            borda,
2856            winner,
2857            rankings: tops.len(),
2858            unanimous_initial,
2859            deliberated,
2860            changed_votes,
2861            unanimous_final,
2862            tie_break,
2863            judges: judges_total,
2864            present,
2865            quorum,
2866            met_quorum,
2867            uncontested,
2868        });
2869        self.state.status = if met_quorum {
2870            RunStatus::Reviewing
2871        } else {
2872            RunStatus::Stalled
2873        };
2874        self.state.save()?;
2875        Ok(())
2876    }
2877
2878    // ------------------------------------------------------------- recover
2879
2880    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2881    /// resumed toward completion once the transient cause clears.
2882    ///
2883    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2884    /// it toward the quorum, which is exactly the set of seats whose absence
2885    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2886    /// not depend on which node happened to hit the limit), or an ordinary
2887    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2888    /// seat is never disturbed.
2889    ///
2890    /// A seat that now answers with a usable ranking is "recovered": its
2891    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2892    /// `tally` counts it present again), and its vote re-collected. A seat that
2893    /// still fails keeps its loss and stays absent.
2894    ///
2895    /// Returns `true` when the re-tally restores the quorum (the run may proceed
2896    /// to review/gate/merge), `false` when it is still below quorum (the run
2897    /// stays `Stalled`, still resumable for a later retry).
2898    #[allow(clippy::too_many_lines)]
2899    async fn recover_stall(&mut self) -> Result<bool> {
2900        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2901        // agent files with `magi task add` name the run that paid for it. The
2902        // prompt overlay is cloned alongside it because the waves borrow it
2903        // while `self` is mutably borrowed by the node's own bookkeeping.
2904        let run_id = self.state.id.clone();
2905        let prompts = self.state.config.prompts.clone();
2906        // Absent seats = quota-lost at any node, or failed outright. Mirroring
2907        // `tally`'s presence test (rather than the old quota-judge/vote filter)
2908        // is what keeps a non-quota collapse — or a quota loss recorded at the
2909        // deliberate node — from being a permanent dead-end on `--resume`.
2910        let quota_seats: BTreeSet<&str> =
2911            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2912        let absent: Vec<String> = self
2913            .state
2914            .judgements
2915            .iter()
2916            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2917            .map(|j| j.seat.clone())
2918            .collect();
2919        if absent.is_empty() {
2920            return Ok(false);
2921        }
2922        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2923        if viable.len() <= 1 {
2924            return Ok(false);
2925        }
2926        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2927        let language = self.state.config.graph.language.clone();
2928        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2929        let sessions = self.state.config.graph.sessions;
2930        let artifacts = agent::artifacts_dir(&self.state.dir());
2931        let root = self.state.worktree_root();
2932        let base_short = short(&self.state.base_commit);
2933        let candidates: Vec<Candidate> = viable.clone();
2934
2935        // Map each absent seat key to its 0-based position in `roles.judges`.
2936        let mut positions: Vec<usize> = absent
2937            .iter()
2938            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2939            .collect();
2940        if positions.is_empty() {
2941            return Ok(false);
2942        }
2943        positions.sort_unstable();
2944        positions.dedup();
2945
2946        // Re-rank the lost seats, one blind prompt each.
2947        let mut judge_jobs = Vec::new();
2948        for &j in &positions {
2949            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2950            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2951            let seat_key = format!("judge-{}", j + 1);
2952            let spec = self.roles.judges[j].clone();
2953            let seat = self.seat(&seat_key, &spec.id);
2954            judge_jobs.push(SeatJob {
2955                spec,
2956                seat,
2957                prompt: prompt::judge(
2958                    &self.state.instruction,
2959                    &views,
2960                    self.roles.judges.len(),
2961                    &base_short,
2962                    &language,
2963                ),
2964                cwd: root.join(seat_key),
2965                timeout,
2966                allow_write: false,
2967                sessions,
2968                artifacts: artifacts.clone(),
2969                stem: format!("judge-{}-recover", j + 1),
2970            });
2971        }
2972
2973        let labels_for_check = labels.clone();
2974        let mut judge_losses = Vec::new();
2975        let retries = self.state.config.graph.retries;
2976        let cache = self.state.config.cache_dir();
2977        let ctx = WaveCtx {
2978            run: &run_id,
2979            node: "judge",
2980            prompts: &prompts,
2981            cache: cache.as_deref(),
2982            round: None,
2983        };
2984        let results = ask_json_wave::<Ranking>(
2985            judge_jobs,
2986            Arc::clone(&self.sem),
2987            retries,
2988            &ctx,
2989            &mut judge_losses,
2990            &mut self.state,
2991            &move |r: &Ranking| r.validate(&labels_for_check),
2992        )
2993        .await;
2994
2995        // Refresh the judgement of every seat that ranked again.
2996        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2997        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
2998            self.state.seats.insert(seat.key.clone(), seat);
2999            let record = &mut self.state.judgements[j];
3000            match res {
3001                Ok((ranking, out)) => {
3002                    record.ranking = ranking.normalized();
3003                    record.reasons = ranking.reasons;
3004                    record.confidence = ranking.confidence;
3005                    record.failed = None;
3006                    record.duration_ms = out.duration_ms;
3007                    recovered.insert(j);
3008                    self.state.event(
3009                        "recover",
3010                        format!("judge {} ranked again after the limit", j + 1),
3011                    );
3012                }
3013                Err(e) => {
3014                    self.state
3015                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3016                }
3017            }
3018        }
3019
3020        // Re-ask the votes of the seats that recovered a ranking.
3021        let mut vote_jobs = Vec::new();
3022        let mut vote_pos: Vec<usize> = Vec::new();
3023        for &j in &recovered {
3024            let seat_key = format!("judge-{}", j + 1);
3025            let spec = self.roles.judges[j].clone();
3026            let seat = self.seat(&seat_key, &spec.id);
3027            let mut text = prompt::final_vote(&labels, &language);
3028            if !has_context(&spec, &seat, sessions) {
3029                text = format!(
3030                    "{}\n\n# Candidates\n\n{}",
3031                    text,
3032                    self.candidate_block(&candidates, &base_short)
3033                );
3034            }
3035            vote_jobs.push(SeatJob {
3036                spec,
3037                seat,
3038                prompt: text,
3039                cwd: root.join(seat_key),
3040                timeout,
3041                allow_write: false,
3042                sessions,
3043                artifacts: artifacts.clone(),
3044                stem: format!("vote-judge-{}-recover", j + 1),
3045            });
3046            vote_pos.push(j);
3047        }
3048        let allowed = labels.clone();
3049        let mut vote_losses = Vec::new();
3050        let vote_retries = self.state.config.graph.retries;
3051        let vote_cache = self.state.config.cache_dir();
3052        let ctx = WaveCtx {
3053            run: &run_id,
3054            node: "vote",
3055            prompts: &prompts,
3056            cache: vote_cache.as_deref(),
3057            round: None,
3058        };
3059        let votes = ask_json_wave::<FinalVote>(
3060            vote_jobs,
3061            Arc::clone(&self.sem),
3062            vote_retries,
3063            &ctx,
3064            &mut vote_losses,
3065            &mut self.state,
3066            &move |v: &FinalVote| match v.label() {
3067                Some(c) if allowed.contains(&c) => Ok(()),
3068                other => bail!("vote {other:?} is not one of {allowed:?}"),
3069            },
3070        )
3071        .await;
3072        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3073            let agent_id = seat.agent.clone();
3074            self.state.seats.insert(seat.key.clone(), seat);
3075            match res {
3076                Ok((v, _)) => {
3077                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3078                        rec.vote = v.label();
3079                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3080                    } else {
3081                        self.state.votes.push(VoteRecord {
3082                            judge: j + 1,
3083                            agent: agent_id,
3084                            vote: v.label(),
3085                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3086                            changed: false,
3087                        });
3088                    }
3089                    self.state.event(
3090                        "recover",
3091                        format!("judge {} voted again after the limit", j + 1),
3092                    );
3093                }
3094                Err(e) => {
3095                    self.state
3096                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3097                }
3098            }
3099        }
3100
3101        // A seat that ranked again is present even if its re-vote failed —
3102        // `tally` falls back to the initial ranking's first choice — so clear
3103        // its quota loss. Seats that still fail keep theirs and stay absent.
3104        let recovered_keys: BTreeSet<String> = recovered
3105            .iter()
3106            .map(|&j| format!("judge-{}", j + 1))
3107            .collect();
3108        self.state
3109            .quota
3110            .retain(|q| !recovered_keys.contains(&q.seat));
3111        // A seat that hit the limit again is a fresh loss, not the old one:
3112        // replace the stale entry so the history stays one-per-seat and the
3113        // daemon can tell this attempt's loss from a previous session's.
3114        for loss in judge_losses.into_iter().chain(vote_losses) {
3115            if recovered_keys.contains(&loss.seat) {
3116                continue;
3117            }
3118            self.state.quota.retain(|q| q.seat != loss.seat);
3119            self.state.quota.push(loss);
3120        }
3121
3122        // Recompute the verdict from the refreshed panel.
3123        self.state.tally = None;
3124        self.tally()?;
3125        Ok(self
3126            .state
3127            .tally
3128            .as_ref()
3129            .map(|t| t.met_quorum)
3130            .unwrap_or(false))
3131    }
3132
3133    // ----------------------------------------------------------------- fold
3134
3135    async fn fold_losers(&mut self) -> Result<()> {
3136        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3137            return Ok(());
3138        };
3139        let repo = self.state.repo.clone();
3140        let mut folded = Vec::new();
3141        for i in 0..self.state.candidates.len() {
3142            let c = &self.state.candidates[i];
3143            if c.label == winner || c.folded {
3144                continue;
3145            }
3146            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3147            git::worktree_remove(&repo, &wt).await.ok();
3148            git::branch_delete(&repo, &branch).await.ok();
3149            self.state.candidates[i].folded = true;
3150            folded.push(label.to_string());
3151        }
3152        // The judges are finished; their checkouts are pure cost from here.
3153        let root = self.state.worktree_root();
3154        for j in 1..=self.roles.judges.len() {
3155            let wt = root.join(format!("judge-{j}"));
3156            if wt.exists() {
3157                git::worktree_remove(&repo, &wt).await.ok();
3158            }
3159        }
3160        // The design-deliberation stage is finished by the time a tally
3161        // exists — same reasoning as the judges above.
3162        if self.state.config.graph.advise {
3163            for k in 1..=self.state.config.graph.advisors {
3164                let wt = root.join(format!("advisor-{k}"));
3165                if wt.exists() {
3166                    git::worktree_remove(&repo, &wt).await.ok();
3167                }
3168            }
3169        }
3170        if !folded.is_empty() {
3171            self.state
3172                .event("fold", format!("folded candidates {}", folded.join(", ")));
3173            self.state.save()?;
3174        }
3175        Ok(())
3176    }
3177
3178    // ------------------------------------------------------------ base sync
3179
3180    /// Land the winner's tree on the current tip of `<remote>/<base>` before
3181    /// anything verifies it.
3182    ///
3183    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
3184    /// read whatever is checked out in the winner's worktree. Left alone that
3185    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
3186    /// it when the run *branched* - and a run takes long enough that the base
3187    /// has usually moved by the time it gets here. A gate that ran there
3188    /// answers "green on the commit this run started from", not "green on
3189    /// what is about to land", and the difference showed up three times in
3190    /// one day as a green run whose merge would have reverted a file another
3191    /// pull request had already landed.
3192    ///
3193    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
3194    /// implementation of the same idea: `land::Step::Rebase` already worked
3195    /// out the rules - throwaway worktree, conflict stops and reports rather
3196    /// than feeding a fixer, nothing runs in the primary tree - and a second
3197    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
3198    /// warns about ("two answers to a question nobody notices until a diff is
3199    /// wrong").
3200    ///
3201    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
3202    /// so it survives a park/resume. A conflict or a push failure sets
3203    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
3204    /// as they were - untouched, for a person to look at - which is also what
3205    /// makes re-entering this function afterwards a no-op instead of a second
3206    /// attempt at the same wall.
3207    async fn sync_to_base(&mut self) -> Result<()> {
3208        if self
3209            .state
3210            .base_sync
3211            .as_ref()
3212            .is_some_and(|s| s.conflict.is_some())
3213        {
3214            return Ok(());
3215        }
3216        let Some(winner) = self.state.winner().cloned() else {
3217            return Ok(());
3218        };
3219
3220        let repo = self.state.repo.clone();
3221        let remote = self.state.config.merge.remote.clone();
3222        let base_branch = self.state.base_branch.clone();
3223        let tracking = format!("{remote}/{base_branch}");
3224
3225        git::fetch(&repo, &remote, &base_branch).await.ok();
3226        // No network, or the remote never had this branch: `resolve_base`
3227        // already treats that as non-fatal at branch time, and a run that got
3228        // this far must not be blocked by it here either.
3229        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3230            return Ok(());
3231        };
3232
3233        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3234        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3235        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3236
3237        if behind == 0 {
3238            self.state.base_sync = Some(BaseSync {
3239                tip,
3240                behind: 0,
3241                attempts,
3242                conflict: None,
3243            });
3244            self.state.save()?;
3245            return Ok(());
3246        }
3247
3248        if attempts >= BASE_SYNC_ROUNDS {
3249            let why = format!(
3250                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3251                 rebase(s); rebasing again would only race it",
3252                winner.branch
3253            );
3254            self.state.status = RunStatus::Blocked;
3255            self.state.base_sync = Some(BaseSync {
3256                tip,
3257                behind,
3258                attempts,
3259                conflict: Some(why.clone()),
3260            });
3261            self.state.event("land", why);
3262            self.state.save()?;
3263            return Ok(());
3264        }
3265
3266        self.state.event(
3267            "land",
3268            format!(
3269                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3270                winner.branch
3271            ),
3272        );
3273        self.state.save()?;
3274
3275        // The remote's copy of the branch, read now and only if the fetch
3276        // really succeeded (a stale tracking ref must never pin a lease). It is
3277        // pushed over after a rebase only when it is a commit this branch
3278        // already contains, by ancestry or by patch (an earlier rebase of ours
3279        // that never reached the remote): anything else is somebody else's work.
3280        let branch_tracking = format!("{remote}/{}", winner.branch);
3281        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3282        let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3283            git::rev_parse(&repo, &branch_tracking).await.ok()
3284        } else {
3285            None
3286        };
3287        // A remote tip this branch does not contain is somebody else's work:
3288        // rebasing would leave a local tip that can never be pushed. Stop
3289        // before touching anything and say so.
3290        if let Some(theirs) = &remote_tip
3291            && !git::is_ancestor(&repo, theirs, &head).await
3292            && !crate::reconcile::origin_missing(&repo, &head, theirs)
3293                .await
3294                .is_ok_and(|missing| missing.is_empty())
3295        {
3296            let why = format!(
3297                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3298                 them",
3299                short(theirs),
3300                winner.branch
3301            );
3302            self.state.status = RunStatus::Blocked;
3303            self.state.base_sync = Some(BaseSync {
3304                tip,
3305                behind,
3306                attempts,
3307                conflict: Some(why.clone()),
3308            });
3309            self.state.event("land", why);
3310            self.state.save()?;
3311            return Ok(());
3312        }
3313
3314        let scratch = self.state.dir().join("base-sync");
3315        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
3316        let attempts = attempts + 1;
3317        match rebased {
3318            Ok(None) => {
3319                // The branch ref moved, but a worktree that already had it
3320                // checked out (the winner's) was not told; sync its index and
3321                // files before anything reads them.
3322                git::sync_to_head(&winner.worktree).await?;
3323                let mut conflict = None;
3324                if let Some(pinned) = &remote_tip {
3325                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3326                    match pushed {
3327                        Ok(o) if o.ok() => self.state.event(
3328                            "land",
3329                            format!("pushed rebased {} to {remote}", winner.branch),
3330                        ),
3331                        Ok(o) => {
3332                            conflict = Some(format!(
3333                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
3334                                winner.branch,
3335                                short(pinned),
3336                                o.stderr.chars().take(600).collect::<String>()
3337                            ));
3338                        }
3339                        Err(e) => {
3340                            conflict = Some(format!(
3341                                "rebased {} locally but could not push it: {e:#}",
3342                                winner.branch
3343                            ));
3344                        }
3345                    }
3346                }
3347                if let Some(why) = &conflict {
3348                    self.state.status = RunStatus::Blocked;
3349                    self.state.event("land", why.clone());
3350                }
3351                self.state.base_sync = Some(BaseSync {
3352                    tip: tip.clone(),
3353                    behind: 0,
3354                    attempts,
3355                    conflict,
3356                });
3357                self.state
3358                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
3359            }
3360            Ok(Some(conflict)) => {
3361                let why = format!(
3362                    "{} conflicts with {tracking} and did not rebase: {}",
3363                    winner.branch,
3364                    conflict.chars().take(600).collect::<String>()
3365                );
3366                self.state.status = RunStatus::Blocked;
3367                self.state.base_sync = Some(BaseSync {
3368                    tip,
3369                    behind,
3370                    attempts,
3371                    conflict: Some(why.clone()),
3372                });
3373                self.state.event("land", why);
3374            }
3375            Err(e) => {
3376                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3377                self.state.status = RunStatus::Blocked;
3378                self.state.base_sync = Some(BaseSync {
3379                    tip,
3380                    behind,
3381                    attempts,
3382                    conflict: Some(why.clone()),
3383                });
3384                self.state.event("land", why);
3385            }
3386        }
3387        self.state.save()?;
3388        Ok(())
3389    }
3390
3391    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
3392    /// last landed the winner on, once it has run, else the commit the run
3393    /// branched from.
3394    ///
3395    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
3396    /// and `vote` all happen before there is a winner to rebase, so they
3397    /// compare every candidate against the branch point on purpose, and a
3398    /// base that moves after they are already done cannot change an answer
3399    /// they already gave.
3400    fn landing_base(&self) -> String {
3401        self.state
3402            .base_sync
3403            .as_ref()
3404            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3405    }
3406
3407    // ------------------------------------------------------- operator fix
3408
3409    /// Route specific, already-recorded review findings to a fixer for a
3410    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
3411    /// entry point.
3412    ///
3413    /// Distinct from `review_loop`'s own fix step in three ways: it never
3414    /// runs a reviewer wave, it never spends review-round budget, and what
3415    /// happened is recorded as an [`OperatorFixRequest`] appended to
3416    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
3417    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
3418    /// and vote must never be rewritten to look like a manufactured blocking
3419    /// verdict.
3420    ///
3421    /// Only meaningful once review has actually concluded: `Ready` (handed
3422    /// off with findings still open, or simply concluded clean while minor
3423    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
3424    /// gate failed). Everything else is refused: a run still in progress
3425    /// should simply be resumed, and a `Merged` run's branch has already
3426    /// landed — reopening *this* run's own record cannot change that, so the
3427    /// answer there is a fresh `magi review <branch>`.
3428    ///
3429    /// A real commit here re-verifies through a fresh, ordinary review-only
3430    /// run on the same branch ([`Self::review`]) rather than reopening this
3431    /// run's own `review_loop`: once any round in this run's history went
3432    /// clean, `review_conclusion` treats that as permanent by design (the
3433    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
3434    /// way to force one more genuine reviewer wave out of *this* run without
3435    /// either rewriting history or weakening that guarantee for every other
3436    /// caller. A review-only run costs nothing extra — no implementation, no
3437    /// judging, no vote — and exercises the exact same review → verify →
3438    /// gate → (human) merge path, unmodified.
3439    pub async fn fix_selected(
3440        &mut self,
3441        ids: &[String],
3442        reason: &str,
3443        allow_stale: bool,
3444    ) -> Result<()> {
3445        let reason = reason.trim();
3446        if reason.is_empty() {
3447            bail!("a fix request needs a reason — that is the operator's own record of why");
3448        }
3449        if ids.is_empty() {
3450            bail!("no finding id given");
3451        }
3452        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3453            bail!(
3454                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3455                 has already concluded — can be given a targeted fix. A run still \
3456                 in progress should simply be resumed; a `merged` run's branch has \
3457                 already landed, so its answer is a fresh `magi review <branch>`, \
3458                 not reopening this run's own record",
3459                self.state.id,
3460                self.state.status.as_str()
3461            );
3462        }
3463        let Some(winner) = self.state.winner().cloned() else {
3464            bail!("run {} has no winning candidate to fix", self.state.id);
3465        };
3466        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3467            bail!(
3468                "branch `{}` no longer exists; this run cannot be extended",
3469                winner.branch
3470            );
3471        }
3472        let home = crate::run::home();
3473        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3474            bail!(
3475                "run {} is currently being worked on by another magi process",
3476                self.state.id
3477            );
3478        }
3479        // Held for the rest of this call, including the follow-up review
3480        // below: two `magi fix` invocations against the same run must not
3481        // both reach the worktree manipulation further down, which would
3482        // otherwise race to remove and recreate the same directory — see
3483        // [`FixClaim`]'s own doc.
3484        let _claim = FixClaim::acquire(&self.state.dir())?;
3485
3486        // Resolve every id before spending anything — an unknown id refuses
3487        // the whole request rather than silently dropping it — and dedup
3488        // while keeping the operator's own order.
3489        let mut seen = BTreeSet::new();
3490        let mut findings = Vec::new();
3491        let mut missing = Vec::new();
3492        for id in ids {
3493            if !seen.insert(id.clone()) {
3494                continue;
3495            }
3496            match self.state.finding(id) {
3497                Some((round, rec, f)) => findings.push(OperatorFixFinding {
3498                    id: f.id.clone(),
3499                    severity: f.severity,
3500                    reviewer_vote: rec.vote,
3501                    round: round.round,
3502                    round_head: round.head.clone(),
3503                    reviewer: rec.reviewer,
3504                    agent: rec.agent.clone(),
3505                    file: f.file.clone(),
3506                    line: f.line,
3507                    title: f.title.clone(),
3508                    detail: f.detail.clone(),
3509                    outcome: OperatorFixOutcome::Pending,
3510                }),
3511                None => missing.push(id.clone()),
3512            }
3513        }
3514        if !missing.is_empty() {
3515            bail!(
3516                "unknown finding id(s): {}; nothing was changed",
3517                missing.join(", ")
3518            );
3519        }
3520
3521        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3522        let stale_details: Vec<(String, String)> = findings
3523            .iter()
3524            .filter(|f| f.round_head != head_at_request)
3525            .map(|f| (f.id.clone(), f.round_head.clone()))
3526            .collect();
3527        let stale = !stale_details.is_empty();
3528        if stale && !allow_stale {
3529            bail!(
3530                "the branch has moved since some finding(s) were raised — {} — now \
3531                 at {}; pass --allow-stale to fix anyway, or re-run review first",
3532                stale_details
3533                    .iter()
3534                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3535                    .collect::<Vec<_>>()
3536                    .join(", "),
3537                short(&head_at_request)
3538            );
3539        }
3540
3541        let request = OperatorFixRequest {
3542            requested_at: Timestamp::now(),
3543            reason: reason.to_owned(),
3544            findings,
3545            head_at_request: head_at_request.clone(),
3546            allow_stale,
3547            stale,
3548            fix: None,
3549            result_head: None,
3550            follow_up_review_run: None,
3551        };
3552        self.state.event(
3553            "fix",
3554            format!(
3555                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3556                request.findings.len(),
3557                request
3558                    .findings
3559                    .iter()
3560                    .map(|f| f.id.as_str())
3561                    .collect::<Vec<_>>()
3562                    .join(", "),
3563            ),
3564        );
3565        // Recorded now, before any worktree work or the fixer call itself —
3566        // and re-saved at each checkpoint below: a crash at any point after
3567        // this (mid fixer call, mid follow-up review) must not lose the fact
3568        // that this was requested, for which findings, and why. Everything
3569        // past this point reads and writes through `request_index` rather
3570        // than a local variable, since `request` itself is moved here.
3571        self.state.operator_fixes.push(request);
3572        self.state.save()?;
3573        let request_index = self.state.operator_fixes.len() - 1;
3574
3575        // A fresh, dedicated worktree for this one call, never the winner's
3576        // own worktree in place: that one may already be gone (folded away),
3577        // and reusing it in place would leave the branch checked out there
3578        // when the follow-up review below tries to check it out again. Freed
3579        // immediately after, either way — but only once confirmed clean:
3580        // `worktree_remove` is a `git worktree remove --force`, which would
3581        // otherwise discard uncommitted work left there by the operator or
3582        // another process before this had a chance to even look at it.
3583        if winner.worktree.exists() {
3584            // Lockfiles a rescue commit withheld stay untracked on purpose and
3585            // are already recorded; they are not the operator's work to protect.
3586            let dirty = git::git(
3587                &winner.worktree,
3588                &["status", "--porcelain", "--untracked-files=all"],
3589            )
3590            .await?;
3591            let only_withheld = dirty.lines().all(|l| {
3592                l.strip_prefix("?? ")
3593                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3594            });
3595            if !only_withheld {
3596                bail!(
3597                    "`{}` has uncommitted changes; refusing to touch it — commit or \
3598                     discard them first",
3599                    winner.worktree.display()
3600                );
3601            }
3602            git::worktree_remove(&self.state.repo, &winner.worktree)
3603                .await
3604                .ok();
3605        }
3606        let fix_worktree = self.state.worktree_root().join("operator-fix");
3607        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3608        git::git(
3609            &self.state.repo,
3610            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3611        )
3612        .await
3613        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3614        if !git::is_clean(&fix_worktree).await? {
3615            git::worktree_remove(&self.state.repo, &fix_worktree)
3616                .await
3617                .ok();
3618            bail!(
3619                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3620                winner.branch
3621            );
3622        }
3623
3624        let run_id = self.state.id.clone();
3625        let prompts = self.state.config.prompts.clone();
3626        let language = self.state.config.graph.language.clone();
3627        let sessions = self.state.config.graph.sessions;
3628        let artifacts = agent::artifacts_dir(&self.state.dir());
3629        let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3630            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3631            _ => (
3632                self.state
3633                    .config
3634                    .agent(&winner.agent)
3635                    .cloned()
3636                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3637                format!("impl-{}", winner.label),
3638            ),
3639        };
3640        let seat = self.seat(&fix_seat_key, &fix_spec.id);
3641        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3642            .findings
3643            .iter()
3644            .map(|f| Finding {
3645                id: f.id.clone(),
3646                severity: f.severity,
3647                file: f.file.clone(),
3648                line: f.line,
3649                title: f.title.clone(),
3650                detail: f.detail.clone(),
3651            })
3652            .collect();
3653        let job = SeatJob {
3654            prompt: prompt::operator_fix(
3655                &self.state.instruction,
3656                &finding_list,
3657                reason,
3658                &stale_details,
3659                &head_at_request,
3660                &language,
3661            ),
3662            spec: fix_spec.clone(),
3663            seat,
3664            cwd: fix_worktree.clone(),
3665            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3666            allow_write: true,
3667            sessions,
3668            artifacts: artifacts.clone(),
3669            stem: "operator-fix".to_owned(),
3670        };
3671        let cache = self.state.config.cache_dir();
3672        let ctx = WaveCtx {
3673            run: &run_id,
3674            node: "fix",
3675            prompts: &prompts,
3676            cache: cache.as_deref(),
3677            round: None,
3678        };
3679        let (seat, out) =
3680            run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3681        let agent_id = seat.agent.clone();
3682
3683        let mut fix = FixRecord {
3684            agent: agent_id,
3685            addressed: Vec::new(),
3686            rejected: Vec::new(),
3687            notes: String::new(),
3688            committed: false,
3689            failed: None,
3690            duration_ms: 0,
3691            continuation: None,
3692        };
3693        let mut final_seat = seat.clone();
3694        match out {
3695            AgentOutcome::Ok(o) => {
3696                fix.duration_ms = o.duration_ms;
3697                let parsed = verdict::extract_json::<FixReport>(&o.text);
3698                let incomplete_reason = match &parsed {
3699                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3700                        "the reply parsed, but it reported a command whose own CLI \
3701                         never confirmed an exit status"
3702                            .to_owned(),
3703                    ),
3704                    Ok(_) => None,
3705                    Err(e) => Some(e.to_string()),
3706                };
3707                match incomplete_reason {
3708                    None => {
3709                        let report = parsed.expect("checked Ok above");
3710                        fix.addressed = report.addressed;
3711                        fix.rejected = report.rejected;
3712                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
3713                    }
3714                    Some(reason) => {
3715                        let (resumed_seat, resolved, failure, cont) = self
3716                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
3717                            .await;
3718                        fix.duration_ms += cont.cumulative_wait_ms;
3719                        fix.continuation = Some(cont);
3720                        final_seat = resumed_seat;
3721                        match resolved {
3722                            Some(report) => {
3723                                fix.addressed = report.addressed;
3724                                fix.rejected = report.rejected;
3725                                fix.notes =
3726                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
3727                            }
3728                            None => fix.failed = failure,
3729                        }
3730                    }
3731                }
3732            }
3733            AgentOutcome::Dropped(o) => {
3734                fix.duration_ms = o.duration_ms;
3735                let why = o
3736                    .dropped
3737                    .as_ref()
3738                    .map(|d| d.why.as_str())
3739                    .unwrap_or("the CLI ended the stream without delivering its answer");
3740                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3741            }
3742            AgentOutcome::Quota(o) => {
3743                self.state.quota.push(QuotaLoss {
3744                    seat: final_seat.key.clone(),
3745                    node: "fix".to_owned(),
3746                    at: Timestamp::now(),
3747                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3748                });
3749                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3750            }
3751            AgentOutcome::Failed(e) => fix.failed = Some(e),
3752        }
3753        if fix.continuation.is_none() {
3754            fix.continuation = Some(ContinuationRecord::not_needed());
3755        }
3756        self.state.seats.insert(final_seat.key.clone(), final_seat);
3757
3758        let rescue_message = format!(
3759            "magi: operator-selected fix ({}) (uncommitted work)",
3760            self.state.operator_fixes[request_index]
3761                .findings
3762                .iter()
3763                .map(|f| f.id.as_str())
3764                .collect::<Vec<_>>()
3765                .join(", ")
3766        );
3767        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
3768            self.state.note_withheld("fix", &r.withheld);
3769        }
3770        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
3771        fix.committed = after != head_at_request;
3772        git::worktree_remove(&self.state.repo, &fix_worktree)
3773            .await
3774            .ok();
3775
3776        self.state.event(
3777            "fix",
3778            match &fix.failed {
3779                Some(reason) => format!(
3780                    "operator fix: adoption report was lost ({reason}); {}",
3781                    if fix.committed {
3782                        "committed"
3783                    } else {
3784                        "NO new commit"
3785                    }
3786                ),
3787                None => format!(
3788                    "operator fix: {} addressed, {} rejected, {}",
3789                    fix.addressed.len(),
3790                    fix.rejected.len(),
3791                    if fix.committed {
3792                        "committed"
3793                    } else {
3794                        "NO new commit"
3795                    }
3796                ),
3797            },
3798        );
3799
3800        // Every selected finding gets an outcome — never left `Pending` once
3801        // the fixer's own turn is over. A report that never came back at all
3802        // marks every one of them `Unreported`, not silently "not addressed":
3803        // quota, a dropped stream, or an exhausted continuation are gaps in
3804        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
3805        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
3806        for f in &mut self.state.operator_fixes[request_index].findings {
3807            f.outcome = if fix.failed.is_some() {
3808                OperatorFixOutcome::Unreported
3809            } else if fix.addressed.contains(&f.id) {
3810                OperatorFixOutcome::Addressed
3811            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
3812                OperatorFixOutcome::Rejected { why: r.why.clone() }
3813            } else {
3814                OperatorFixOutcome::Unreported
3815            };
3816        }
3817
3818        let committed = fix.committed;
3819        if committed {
3820            self.state.operator_fixes[request_index].result_head = Some(after.clone());
3821        }
3822        self.state.operator_fixes[request_index].fix = Some(fix);
3823        // Saved again now that the fixer's own outcome is final, on top of
3824        // the save right after the request was first pushed above.
3825        self.state.save()?;
3826
3827        if committed {
3828            self.state.event(
3829                "fix",
3830                format!(
3831                    "operator fix committed {}; opening a follow-up review-only run",
3832                    short(&after)
3833                ),
3834            );
3835            match Self::review(&self.state.repo, &winner.branch, self.state.config.clone()).await {
3836                Ok(mut follow_up) => {
3837                    follow_up.state.event(
3838                        "start",
3839                        format!(
3840                            "requested by an operator fix on run {} for finding(s) {}",
3841                            self.state.id,
3842                            self.state.operator_fixes[request_index]
3843                                .findings
3844                                .iter()
3845                                .map(|f| f.id.as_str())
3846                                .collect::<Vec<_>>()
3847                                .join(", "),
3848                        ),
3849                    );
3850                    follow_up.state.save()?;
3851                    let follow_up_id = follow_up.state.id.clone();
3852                    if let Err(e) = follow_up.execute().await {
3853                        self.state.event(
3854                            "fix",
3855                            format!(
3856                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
3857                            ),
3858                        );
3859                    }
3860                    self.state.operator_fixes[request_index].follow_up_review_run =
3861                        Some(follow_up_id);
3862                }
3863                Err(e) => {
3864                    self.state.event(
3865                        "fix",
3866                        format!("committed the fix but could not open a follow-up review: {e:#}"),
3867                    );
3868                }
3869            }
3870            self.state.save()?;
3871        }
3872
3873        Ok(())
3874    }
3875
3876    // --------------------------------------------------------------- review
3877
3878    /// The agent and seat key that fix the winner's tree: the configured
3879    /// fixer, else the winner's own implementer seat, whose conversation
3880    /// continues now that the competition is over. Shared by the review loop
3881    /// and the gate-fix round so both talk to the same seat.
3882    fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
3883        match &self.roles.fixer {
3884            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3885            _ => (
3886                self.state
3887                    .config
3888                    .agent(&winner.agent)
3889                    .cloned()
3890                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3891                format!("impl-{}", winner.label),
3892            ),
3893        }
3894    }
3895
3896    async fn review_loop(&mut self) -> Result<()> {
3897        // A base that would not rebase is a person's decision, not a review
3898        // round: nothing here would change the answer, and reviewers and a
3899        // fixer would be spending real budget on a tree that cannot land
3900        // regardless of what they find.
3901        if self
3902            .state
3903            .base_sync
3904            .as_ref()
3905            .is_some_and(|s| s.conflict.is_some())
3906        {
3907            return Ok(());
3908        }
3909        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3910        // agent files with `magi task add` name the run that paid for it. The
3911        // prompt overlay is cloned alongside it because the waves borrow it
3912        // while `self` is mutably borrowed by the node's own bookkeeping.
3913        let run_id = self.state.id.clone();
3914        let prompts = self.state.config.prompts.clone();
3915        let Some(winner) = self.state.winner().cloned() else {
3916            return Ok(());
3917        };
3918        let max_rounds = self.state.config.graph.review_rounds;
3919        // A clean round, an exhausted round budget, or a stalled tree (see
3920        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
3921        // they are recorded — recomputed here, not read off `status`, so a
3922        // reentry into a run that already stopped restates the identical
3923        // verdict instead of silently handing back whatever an earlier node
3924        // in this same walk clobbered `status` to (a solo-candidate
3925        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
3926        // below runs an empty range once the budget is spent, and would
3927        // otherwise fall through without touching `status` at all.
3928        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
3929            self.state.status = status;
3930            self.state.save()?;
3931            return Ok(());
3932        }
3933        self.state.status = RunStatus::Reviewing;
3934        // A last recorded round whose own verification never resolved
3935        // (`ResourceBlocked` — the shared build cache, not the patch) is
3936        // never a concluded round, whatever the round budget says: starting
3937        // a fresh round on top of it would spend a whole new reviewer wave
3938        // re-reading an unchanged patch instead of just retrying the one
3939        // check that actually needs it, and once the budget is spent the
3940        // loop below has nothing left to do at all (its range is empty).
3941        // Retry that check directly instead, exactly the same retry
3942        // `stop_reviewing` already does for its own catch-up case.
3943        if self
3944            .state
3945            .reviews
3946            .last()
3947            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
3948        {
3949            let shell = self.state.config.shell();
3950            return self
3951                .stop_reviewing(
3952                    "the last round's own verification never resolved",
3953                    &shell,
3954                    &winner.worktree,
3955                )
3956                .await;
3957        }
3958
3959        let repo = self.state.repo.clone();
3960        let root = self.state.worktree_root();
3961        let language = self.state.config.graph.language.clone();
3962        let sessions = self.state.config.graph.sessions;
3963        let artifacts = agent::artifacts_dir(&self.state.dir());
3964        let base = self.landing_base();
3965        let base_short = short(&base);
3966        let reviewers = self.roles.reviewers.clone();
3967        let shell = self.state.config.shell();
3968
3969        for round in (self.state.reviews.len() + 1)..=max_rounds {
3970            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3971            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
3972            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
3973            // The prior round's own record, already persisted — never a
3974            // hand-carried variable of just its failing output: that is
3975            // exactly what let a round's e2e result drift out of sync with
3976            // which commit it was actually about (see `SCHEMA`'s doc for
3977            // schema 8). Judged against `head`, the commit reviewers are
3978            // about to look at now, so the summary always reads as "an
3979            // earlier head" here — this round's own patch has not been
3980            // checked yet.
3981            let prev_verification = self
3982                .state
3983                .reviews
3984                .last()
3985                .and_then(|r| r.verification_summary(&head));
3986
3987            // Each reviewer gets its own detached checkout of exactly this
3988            // commit: nobody can perturb the winner's tree, and the fixer can
3989            // keep working without racing a reviewer.
3990            let mut jobs = Vec::new();
3991            for (r, spec) in reviewers.iter().cloned().enumerate() {
3992                let wt = root.join(format!("review-{}", r + 1));
3993                if wt.exists() {
3994                    git::reset_detached(&wt, &head).await?;
3995                } else {
3996                    git::worktree_add_detached(&repo, &wt, &head).await?;
3997                }
3998                let seat_key = format!("review-{}", r + 1);
3999                let seat = self.seat(&seat_key, &spec.id);
4000                jobs.push(SeatJob {
4001                    prompt: prompt::review(&prompt::ReviewCtx {
4002                        instruction: &self.state.instruction,
4003                        branch: &winner.branch,
4004                        base_short: &base_short,
4005                        stat: &stat,
4006                        patch: &patch,
4007                        verification: prev_verification.as_ref(),
4008                        reviewers: reviewers.len(),
4009                        round,
4010                        rounds: max_rounds,
4011                        // A review-only run has no rankings, so nothing
4012                        // competed for this patch and the reviewer is told so.
4013                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4014                        lens: Lens::for_seat(r),
4015                        language: &language,
4016                    }),
4017                    spec,
4018                    seat,
4019                    cwd: wt,
4020                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4021                    allow_write: false,
4022                    sessions,
4023                    artifacts: artifacts.clone(),
4024                    stem: format!("review-{round}-{}", r + 1),
4025                });
4026            }
4027
4028            self.state.event(
4029                "review",
4030                format!(
4031                    "round {round}: {} reviewers on {}",
4032                    jobs.len(),
4033                    short(&head)
4034                ),
4035            );
4036            let mut quota_losses = Vec::new();
4037            let review_retries = self.state.config.graph.retries;
4038            let review_cache = self.state.config.cache_dir();
4039            let ctx = WaveCtx {
4040                run: &run_id,
4041                node: "review",
4042                prompts: &prompts,
4043                cache: review_cache.as_deref(),
4044                round: Some(round),
4045            };
4046            let results = ask_json_wave::<Review>(
4047                jobs,
4048                Arc::clone(&self.sem),
4049                review_retries,
4050                &ctx,
4051                &mut quota_losses,
4052                &mut self.state,
4053                &|_: &Review| Ok(()),
4054            )
4055            .await;
4056            // Counted before the move below: how many of *this* round's
4057            // reviewer seats were lost to their own rate limit, as opposed to
4058            // a crash, a timeout, or unparsable output — see `round_is_clean`.
4059            let round_quota_missing = quota_losses.len();
4060            self.state.quota.extend(quota_losses);
4061
4062            let mut records = Vec::new();
4063            let mut all_findings = Vec::new();
4064            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4065                let agent_id = seat.agent.clone();
4066                self.state.seats.insert(seat.key.clone(), seat);
4067                let mut record = ReviewRecord {
4068                    reviewer: r + 1,
4069                    agent: agent_id,
4070                    summary: String::new(),
4071                    findings: Vec::new(),
4072                    vote: None,
4073                    failed: None,
4074                    duration_ms: 0,
4075                    // Set for both outcomes: `failed: Some(_)` with
4076                    // `attempts > 0` is a seat every retry still lost, not a
4077                    // recovered one — only `failed: None` with `attempts > 0`
4078                    // reads as "answered after a nudge" (see this field's own
4079                    // doc).
4080                    attempts,
4081                };
4082                match res {
4083                    Ok((review, out)) => {
4084                        // Sanitized here, at the point every other piece of
4085                        // agent prose in this file is (candidate summaries,
4086                        // deliberation turns, vote reasons): a reviewer's own
4087                        // words are the one thing about it that could name
4088                        // it, and reconsideration below broadcasts this same
4089                        // summary and these same findings to every other
4090                        // seat on the panel.
4091                        record.summary =
4092                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
4093                        record.vote = Some(review.vote);
4094                        record.duration_ms = out.duration_ms;
4095                        for (n, mut f) in review.findings.into_iter().enumerate() {
4096                            // ids are magi's, never the agent's: the fixer's
4097                            // adoption report is keyed by them.
4098                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4099                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4100                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4101                            // `file` is agent-supplied prose too, never
4102                            // checked against the real tree — the same
4103                            // exposure `title`/`detail` above have, just in
4104                            // a field easy to forget because it looks like a
4105                            // path rather than free text.
4106                            f.file = f
4107                                .file
4108                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4109                            all_findings.push(f.clone());
4110                            record.findings.push(f);
4111                        }
4112                        self.state.event(
4113                            "review",
4114                            format!(
4115                                "round {round}: reviewer {} voted {} with {} finding(s)",
4116                                r + 1,
4117                                review.vote.label(),
4118                                record.findings.len()
4119                            ),
4120                        );
4121                    }
4122                    Err(e) => {
4123                        record.failed = Some(e.to_string());
4124                        self.state.event(
4125                            "review",
4126                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4127                        );
4128                    }
4129                }
4130                records.push(record);
4131            }
4132
4133            // Tally the round's votes and, if they split, spend the one
4134            // round of reconsideration the split -> deliberate -> revote
4135            // shape `judge`/`vote` use for the panel, sized down to what a
4136            // read-only review round can afford: one round, and a revote
4137            // rather than an argument, because the panel already wrote its
4138            // reasoning down as findings the first time around.
4139            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4140            let vote_split =
4141                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4142            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4143            if vote_split {
4144                self.state.event(
4145                    "review",
4146                    format!(
4147                        "round {round}: votes split ({}) — one round of reconsideration",
4148                        initial_votes
4149                            .iter()
4150                            .map(|v| v.label())
4151                            .collect::<Vec<_>>()
4152                            .join(", ")
4153                    ),
4154                );
4155                // Seats read every seat's findings and votes, still numbered
4156                // and never named — the same anonymity `review` itself keeps.
4157                let panel: Vec<ReviewSeatReport<'_>> = records
4158                    .iter()
4159                    .filter_map(|r| {
4160                        r.vote.map(|vote| ReviewSeatReport {
4161                            reviewer: r.reviewer,
4162                            vote,
4163                            summary: &r.summary,
4164                            findings: &r.findings,
4165                        })
4166                    })
4167                    .collect();
4168
4169                let mut jobs = Vec::new();
4170                let mut seats_at = Vec::new();
4171                for (r, spec) in reviewers.iter().cloned().enumerate() {
4172                    // A seat with no initial vote has nothing to reconsider
4173                    // from and stays absent, the same as it stayed absent
4174                    // from `panel` above.
4175                    if records[r].vote.is_none() {
4176                        continue;
4177                    }
4178                    let wt = root.join(format!("review-{}", r + 1));
4179                    let seat_key = format!("review-{}", r + 1);
4180                    let seat = self.seat(&seat_key, &spec.id);
4181                    // A seat with no live session has already forgotten the
4182                    // initial review's prompt — restate the patch it is
4183                    // voting on, the same as `deliberate`/`vote` do for a
4184                    // judge in the same position.
4185                    let patch_ctx = if has_context(&spec, &seat, sessions) {
4186                        None
4187                    } else {
4188                        Some(ReviewPatch {
4189                            branch: &winner.branch,
4190                            base_short: &base_short,
4191                            stat: &stat,
4192                            patch: &patch,
4193                        })
4194                    };
4195                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
4196                        instruction: &self.state.instruction,
4197                        reviewer: r + 1,
4198                        lens: Lens::for_seat(r),
4199                        panel: &panel,
4200                        patch: patch_ctx,
4201                        round,
4202                        rounds: max_rounds,
4203                        language: &language,
4204                    });
4205                    jobs.push(SeatJob {
4206                        prompt,
4207                        spec,
4208                        seat,
4209                        cwd: wt,
4210                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4211                        allow_write: false,
4212                        sessions,
4213                        artifacts: artifacts.clone(),
4214                        stem: format!("review-{round}-reconsider-{}", r + 1),
4215                    });
4216                    seats_at.push(r);
4217                }
4218
4219                let mut recon_quota_losses = Vec::new();
4220                let recon_cache = self.state.config.cache_dir();
4221                let recon_ctx = WaveCtx {
4222                    run: &run_id,
4223                    node: "review",
4224                    prompts: &prompts,
4225                    cache: recon_cache.as_deref(),
4226                    round: Some(round),
4227                };
4228                let recon_results = ask_json_wave::<ReviewRevote>(
4229                    jobs,
4230                    Arc::clone(&self.sem),
4231                    review_retries,
4232                    &recon_ctx,
4233                    &mut recon_quota_losses,
4234                    &mut self.state,
4235                    &|_: &ReviewRevote| Ok(()),
4236                )
4237                .await;
4238                self.state.quota.extend(recon_quota_losses);
4239
4240                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4241                    let agent_id = seat.agent.clone();
4242                    self.state.seats.insert(seat.key.clone(), seat);
4243                    let mut rec = ReviewRevoteRecord {
4244                        reviewer: r + 1,
4245                        agent: agent_id,
4246                        vote: None,
4247                        reason: String::new(),
4248                        failed: None,
4249                    };
4250                    match res {
4251                        Ok((rv, _)) => {
4252                            rec.vote = Some(rv.vote);
4253                            rec.reason =
4254                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4255                            self.state.event(
4256                                "review",
4257                                format!(
4258                                    "round {round}: reviewer {} revoted {}",
4259                                    r + 1,
4260                                    rv.vote.label()
4261                                ),
4262                            );
4263                        }
4264                        Err(e) => {
4265                            rec.failed = Some(e.to_string());
4266                            self.state.event(
4267                                "review",
4268                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4269                            );
4270                        }
4271                    }
4272                    reconsideration.push(rec);
4273                }
4274            } else if initial_votes.len() > 1 {
4275                self.state.event(
4276                    "review",
4277                    format!(
4278                        "round {round}: votes agreed ({}) — no reconsideration",
4279                        initial_votes[0].label()
4280                    ),
4281                );
4282            }
4283
4284            // The final vote per seat is its revote where reconsideration
4285            // ran and answered, its initial vote otherwise — the same
4286            // fallback `tally` uses for a judge whose private vote failed.
4287            let final_votes: Vec<ReviewVote> = records
4288                .iter()
4289                .filter_map(|r| {
4290                    reconsideration
4291                        .iter()
4292                        .find(|rv| rv.reviewer == r.reviewer)
4293                        .and_then(|rv| rv.vote)
4294                        .or(r.vote)
4295                })
4296                .collect();
4297            let round_verdict = ReviewVote::worst(final_votes);
4298
4299            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4300            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4301            // A round that already has a blocking finding and a round left to
4302            // try is going back to the fixer no matter what `verify.e2e`
4303            // says, so running it first only spends the loop's slowest step
4304            // (minutes, for a Rust repo's full test suite) on a head about
4305            // to be rewritten. Deferred, never skipped: `verify.e2e` still
4306            // runs once a round has no blocking findings left (see
4307            // `round_is_clean`, which a deferred — empty — `e2e` can never
4308            // satisfy since `blocking` is nonzero whenever this branch is
4309            // taken), and `stop_reviewing` forces a real run before it will
4310            // ever read a deferred round as green.
4311            let defer_e2e =
4312                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4313            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4314                let reason =
4315                    format!("{blocking} blocking finding(s) already required a fix this round");
4316                self.state.event(
4317                    "verify",
4318                    format!(
4319                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4320                         {}); it will run once a round has none left",
4321                        short(&head)
4322                    ),
4323                );
4324                (Vec::new(), false, true, Some(reason))
4325            } else {
4326                let e2e_commands = self.state.config.verify.e2e.clone();
4327                let cache_dir = self.state.config.cache_dir();
4328                let context = format!("round {round}");
4329                let (e2e, verify_retried) = with_cache_lease(
4330                    &mut self.state,
4331                    cache_dir.as_deref(),
4332                    "e2e",
4333                    "e2e",
4334                    &winner.worktree,
4335                    &head,
4336                    verify_timeout,
4337                    &context,
4338                    |state, budget| {
4339                        let shell = shell.clone();
4340                        let e2e_commands = e2e_commands.clone();
4341                        let worktree = winner.worktree.clone();
4342                        let context = context.clone();
4343                        async move {
4344                            run_e2e_with_retry(
4345                                state,
4346                                &shell,
4347                                &e2e_commands,
4348                                &worktree,
4349                                budget,
4350                                &context,
4351                            )
4352                            .await
4353                        }
4354                    },
4355                )
4356                .await;
4357                (e2e, verify_retried, false, None)
4358            };
4359
4360            let expected = records.len();
4361            let answered = records.iter().filter(|r| r.failed.is_none()).count();
4362            let incomplete = answered < expected;
4363            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4364            let policy = self.state.config.graph.incomplete_review;
4365            let clean = round_is_clean(
4366                blocking,
4367                e2e_ok,
4368                answered,
4369                expected,
4370                round_quota_missing,
4371                policy,
4372            );
4373
4374            let mut round_record = ReviewRound {
4375                round,
4376                head: head.clone(),
4377                verified_head: None,
4378                verified_at: None,
4379                reviews: records,
4380                e2e,
4381                verify_retried,
4382                e2e_deferred,
4383                e2e_defer_reason,
4384                fix: None,
4385                blocking,
4386                answered,
4387                expected,
4388                clean,
4389                progressed: false,
4390                vote_split,
4391                reconsideration,
4392                verdict: round_verdict,
4393            };
4394            // Which commit and when magi actually attempted to check —
4395            // known the moment a command was dispatched against `head`,
4396            // whether or not it finished: a resource-blocked attempt still
4397            // targeted a specific commit at a specific time, and leaving
4398            // that unrecorded is exactly what made `verification_summary`
4399            // report a fresh attempt as "commit unknown ... recorded before
4400            // this was tracked", indistinguishable from a genuinely old,
4401            // untracked record. Only a deferred or unconfigured round never
4402            // ran at all and has nothing to record — see
4403            // `ReviewRound::verified_head`'s own doc.
4404            if !matches!(
4405                round_record.e2e_status(),
4406                E2eStatus::Deferred | E2eStatus::NotConfigured
4407            ) {
4408                round_record.verified_head = Some(head.clone());
4409                round_record.verified_at = Some(Timestamp::now());
4410            }
4411            let this_round_verification = round_record.verification_summary(&head);
4412
4413            if incomplete {
4414                let missing: Vec<String> = round_record
4415                    .reviews
4416                    .iter()
4417                    .filter(|r| r.failed.is_some())
4418                    .map(|r| format!("review-{}", r.reviewer))
4419                    .collect();
4420                self.state.event(
4421                    "review",
4422                    format!(
4423                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4424                        missing.join(", ")
4425                    ),
4426                );
4427            }
4428
4429            if clean {
4430                self.state.event(
4431                    "review",
4432                    if incomplete && policy == IncompleteReviewPolicy::Warn {
4433                        format!(
4434                            "round {round}: clean (warn policy, incomplete panel) — no \
4435                             blocking findings from the seats that answered, verification green"
4436                        )
4437                    } else if incomplete {
4438                        format!(
4439                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4440                             quorum) — no blocking findings from the seats that answered, \
4441                             verification green",
4442                            expected - answered
4443                        )
4444                    } else {
4445                        format!("round {round}: clean — no blocking findings, verification green")
4446                    },
4447                );
4448                self.state.reviews.push(round_record);
4449                self.state.status = RunStatus::Gating;
4450                self.state.save()?;
4451                return Ok(());
4452            }
4453
4454            // Nothing was raised and verification passed, but not every seat
4455            // answered and `round_is_clean` still refused to call it clean —
4456            // either a seat is missing for a reason other than its own quota
4457            // (a crash, a timeout, unparsable output — worth another try), or
4458            // every seat that could have answered lost its quota and nobody
4459            // is left to decide on: re-review rather than send the fixer
4460            // after a round with nothing to fix.
4461            if incomplete && blocking == 0 && e2e_ok {
4462                self.state.reviews.push(round_record);
4463                self.state.save()?;
4464                if round == max_rounds {
4465                    self.state.status = RunStatus::Blocked;
4466                    self.state.event(
4467                        "review",
4468                        format!(
4469                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4470                             refusing to call it clean",
4471                            expected - answered
4472                        ),
4473                    );
4474                    return Ok(());
4475                }
4476                continue;
4477            }
4478
4479            // Nothing for the fixer to act on (`blocking == 0`) and the only
4480            // reason this round is not clean is that magi itself never got
4481            // a command to run — the shared build cache, not the patch (see
4482            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
4483            // the fixer would invite a change to appease contention that has
4484            // nothing to do with the diff, and would leave this attempt
4485            // sitting in the next round's prompt as if it were about an
4486            // earlier, superseded commit rather than what it actually is:
4487            // the same head, still waiting to be checked. Wait for it the
4488            // same way the final round's own contention is already handled,
4489            // whatever round this happens to be.
4490            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4491                self.state.reviews.push(round_record);
4492                return self
4493                    .stop_reviewing(
4494                        "the round's own verification could not run",
4495                        &shell,
4496                        &winner.worktree,
4497                    )
4498                    .await;
4499            }
4500
4501            if round == max_rounds {
4502                self.state.reviews.push(round_record);
4503                return self
4504                    .stop_reviewing(
4505                        &format!(
4506                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4507                        ),
4508                        &shell,
4509                        &winner.worktree,
4510                    )
4511                    .await;
4512            }
4513
4514            // Fix. The winner's own implementer seat continues its conversation:
4515            // the competition is over, so context is pure benefit now.
4516            let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4517            let seat = self.seat(&fix_seat_key, &fix_spec.id);
4518            let blocking_findings: Vec<_> = all_findings
4519                .iter()
4520                .filter(|f| f.severity.blocks())
4521                .cloned()
4522                .collect();
4523            let job = SeatJob {
4524                prompt: prompt::fix(
4525                    &self.state.instruction,
4526                    &blocking_findings,
4527                    this_round_verification.as_ref(),
4528                    round,
4529                    max_rounds,
4530                    &language,
4531                ),
4532                spec: fix_spec.clone(),
4533                seat,
4534                cwd: winner.worktree.clone(),
4535                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4536                allow_write: true,
4537                sessions,
4538                artifacts: artifacts.clone(),
4539                stem: format!("fix-{round}"),
4540            };
4541            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4542            let cache = self.state.config.cache_dir();
4543            let ctx = WaveCtx {
4544                run: &run_id,
4545                node: "fix",
4546                prompts: &prompts,
4547                cache: cache.as_deref(),
4548                round: Some(round),
4549            };
4550            let (seat, out) =
4551                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4552            let agent_id = seat.agent.clone();
4553
4554            let mut fix = FixRecord {
4555                agent: agent_id,
4556                addressed: Vec::new(),
4557                rejected: Vec::new(),
4558                notes: String::new(),
4559                committed: false,
4560                failed: None,
4561                duration_ms: 0,
4562                continuation: None,
4563            };
4564            let mut continuation = ContinuationRecord::not_needed();
4565            let mut final_seat = seat.clone();
4566            match out {
4567                AgentOutcome::Ok(o) => {
4568                    fix.duration_ms = o.duration_ms;
4569                    let parsed = verdict::extract_json::<FixReport>(&o.text);
4570                    // A parsed report standing next to a command this same
4571                    // reply's own CLI never confirmed the exit status of is
4572                    // not a resolved answer — the identical `CommandEvidence`
4573                    // `state.jobs` renders, read here instead of only on
4574                    // display, per the completion judgment and the shown
4575                    // record needing to agree.
4576                    let incomplete_reason = match &parsed {
4577                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4578                            "the reply parsed, but it reported a command whose own CLI never \
4579                             confirmed an exit status"
4580                                .to_owned(),
4581                        ),
4582                        Ok(_) => None,
4583                        Err(e) => Some(e.to_string()),
4584                    };
4585                    match incomplete_reason {
4586                        None => {
4587                            let report = parsed.expect("checked Ok above");
4588                            fix.addressed = report.addressed;
4589                            fix.rejected = report.rejected;
4590                            fix.notes =
4591                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
4592                        }
4593                        Some(reason) => {
4594                            let (resumed_seat, resolved, failure, cont) = self
4595                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4596                                .await;
4597                            fix.duration_ms += cont.cumulative_wait_ms;
4598                            continuation = cont;
4599                            final_seat = resumed_seat;
4600                            match resolved {
4601                                Some(report) => {
4602                                    fix.addressed = report.addressed;
4603                                    fix.rejected = report.rejected;
4604                                    fix.notes = blind::sanitize_prose(
4605                                        &report.notes,
4606                                        &self.state.config.blind,
4607                                    );
4608                                }
4609                                None => fix.failed = failure,
4610                            }
4611                        }
4612                    }
4613                }
4614                // The CLI's raw error JSON is not a fix report to parse.
4615                AgentOutcome::Dropped(o) => {
4616                    fix.duration_ms = o.duration_ms;
4617                    let why = o
4618                        .dropped
4619                        .as_ref()
4620                        .map(|d| d.why.as_str())
4621                        .unwrap_or("the CLI ended the stream without delivering its answer");
4622                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4623                }
4624                AgentOutcome::Quota(o) => {
4625                    self.state.quota.push(QuotaLoss {
4626                        seat: final_seat.key.clone(),
4627                        node: "fix".to_owned(),
4628                        at: Timestamp::now(),
4629                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4630                    });
4631                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4632                }
4633                AgentOutcome::Failed(e) => fix.failed = Some(e),
4634            }
4635            fix.continuation = Some(continuation);
4636            self.state.seats.insert(final_seat.key.clone(), final_seat);
4637            if let Ok(r) = git::rescue_commit(
4638                &winner.worktree,
4639                &format!("magi: review round {round} fixes (uncommitted work)"),
4640            )
4641            .await
4642            {
4643                self.state.note_withheld("fix", &r.withheld);
4644            }
4645            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
4646            fix.committed = after != before;
4647            // Judged by what `git` says moved against base, never by the
4648            // fixer's own `addressed`/`rejected` count — see
4649            // `ReviewRound::progressed`. Propagated with `?`, the same as the
4650            // `patch` snapshot above: swallowing this error would default
4651            // `diff_after` to empty, which almost always differs from a
4652            // non-empty `patch` and reads as "progressed" — exactly backwards
4653            // for a `git` failure the stagnation check cannot see through.
4654            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
4655            let progressed = diff_after != patch;
4656            let commit_note = if fix.committed {
4657                "committed"
4658            } else {
4659                "NO new commit"
4660            };
4661            let tree_note = if progressed {
4662                "changed vs base"
4663            } else {
4664                "unchanged vs base"
4665            };
4666            self.state.event(
4667                "fix",
4668                match &fix.failed {
4669                    // Distinct on purpose from "0 addressed, 0 rejected": the
4670                    // fixer's own diff still landed (blocking counts do keep
4671                    // falling round over round), only its adoption report did
4672                    // not come back, so this must never read like every
4673                    // finding was reviewed and declined.
4674                    Some(reason) => {
4675                        format!(
4676                            "round {round}: fixer's adoption report was lost ({reason}); \
4677                             {commit_note}, tree {tree_note}"
4678                        )
4679                    }
4680                    None => format!(
4681                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
4682                         {tree_note}{}",
4683                        fix.addressed.len(),
4684                        fix.rejected.len(),
4685                        if continuation.outcome == ContinuationOutcome::Resumed {
4686                            format!(
4687                                " (adoption report recovered after {} continuation(s))",
4688                                continuation.attempts
4689                            )
4690                        } else {
4691                            String::new()
4692                        },
4693                    ),
4694                },
4695            );
4696            round_record.fix = Some(fix);
4697            round_record.progressed = progressed;
4698            self.state.reviews.push(round_record);
4699            self.state.save()?;
4700
4701            // The fixer's own report never came back this round, even after
4702            // `continue_fix_report`'s own budget was spent on it — not an
4703            // ordinary "no report" (dropped stream, quota, plain failure),
4704            // which already reads that way and is left to the existing round
4705            // budget. Stopping here, rather than opening another round, is
4706            // what keeps a next reviewer/fixer wave from ever being
4707            // dispatched onto `winner.worktree` while whatever the seat's
4708            // last call may still have running there is unaccounted for: no
4709            // process liveness check exists (and none is being added — see
4710            // AGENTS.md/this task's own scope), so the only way to honour
4711            // "nothing starts before a valid report returns" is to not start
4712            // anything further on this worktree from this run at all.
4713            if matches!(
4714                continuation.outcome,
4715                ContinuationOutcome::Exhausted
4716                    | ContinuationOutcome::QuotaLost
4717                    | ContinuationOutcome::NoSession
4718            ) {
4719                return self
4720                    .stop_reviewing(
4721                        "the fixer's adoption report never came back, even after resuming its \
4722                         own seat; refusing to start another round against the same worktree \
4723                         while that is unresolved",
4724                        &shell,
4725                        &winner.worktree,
4726                    )
4727                    .await;
4728            }
4729
4730            let streak = self
4731                .state
4732                .reviews
4733                .iter()
4734                .rev()
4735                .take_while(|r| !r.progressed)
4736                .count();
4737            if streak >= STAGNANT_LIMIT {
4738                return self
4739                    .stop_reviewing(
4740                        &format!(
4741                            "the tree has not moved against base for {streak} round(s) in a row"
4742                        ),
4743                        &shell,
4744                        &winner.worktree,
4745                    )
4746                    .await;
4747            }
4748        }
4749        Ok(())
4750    }
4751
4752    /// Decide, from the last recorded round's own verification, whether
4753    /// stopping the review loop is a hand-off or a genuine block.
4754    ///
4755    /// Called once the loop has given up trying — the round budget is spent,
4756    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
4757    /// findings still open, never while a round is still clean or the
4758    /// incomplete-panel case handled inline above. Gate and e2e are facts
4759    /// about the tree; a lingering review finding is an opinion, and this
4760    /// workload's own `magi stats` puts reviewer precision low enough
4761    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
4762    /// must not by itself stand between a green, verified change and the
4763    /// human who decides what to do with it. A red e2e is not an opinion, so
4764    /// that case still blocks, with the failing command and a tail of its
4765    /// output recorded here rather than left in `run.json` for someone to go
4766    /// find.
4767    ///
4768    /// A round that deferred its own e2e (see [`Config::graph`]'s
4769    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
4770    /// only because nothing ran, and treating an empty list as a passing one
4771    /// here is exactly the "deferred painted green" bug this function exists
4772    /// to not have. When the last round's own verification never resolved —
4773    /// deferred on purpose, or a real attempt the shared build cache blocked
4774    /// — this makes (or retries) the real run, on the actual worktree this
4775    /// loop is about to stop touching, before deciding anything. A
4776    /// resource-blocked attempt is likewise never read as either green or
4777    /// red: it is evidence about the machine, not the patch (see
4778    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
4779    /// blocked cache leaves this call without deciding rather than guessing
4780    /// — the caller retries on a later reentry.
4781    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
4782        let round_idx = self.state.reviews.len() - 1;
4783        // A deferred round and a resource-blocked one are the same shape
4784        // here: neither has a real result yet, and both get one more
4785        // attempt. Read off `e2e_status` — the single source for this —
4786        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
4787        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
4788        // still retries instead of being read as a settled result the
4789        // instant it stops being empty.
4790        let needs_catchup_run = matches!(
4791            self.state.reviews[round_idx].e2e_status(),
4792            E2eStatus::Deferred | E2eStatus::ResourceBlocked
4793        );
4794        if needs_catchup_run {
4795            let round = self.state.reviews[round_idx].round;
4796            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4797            let commands = self.state.config.verify.e2e.clone();
4798            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
4799            let cache_dir = self.state.config.cache_dir();
4800            let context = format!(
4801                "round {round}: verification unresolved, catching up before the final decision"
4802            );
4803            let (outcomes, verify_retried) = with_cache_lease(
4804                &mut self.state,
4805                cache_dir.as_deref(),
4806                "e2e",
4807                "e2e",
4808                worktree,
4809                &attempted_head,
4810                timeout,
4811                &context,
4812                |state, budget| {
4813                    let shell = shell.to_vec();
4814                    let commands = commands.clone();
4815                    let context = context.clone();
4816                    async move {
4817                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
4818                            .await
4819                    }
4820                },
4821            )
4822            .await;
4823            let last = &mut self.state.reviews[round_idx];
4824            last.e2e = outcomes;
4825            last.verify_retried = verify_retried;
4826            // Always the commit and time this attempt actually targeted,
4827            // whether or not it happens to equal the reviewed `head` and
4828            // whether or not a command finished — see
4829            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
4830            // attempt is recorded too, so a later reader sees "attempted
4831            // again at T2" rather than silence.
4832            last.verified_head = Some(attempted_head);
4833            last.verified_at = Some(Timestamp::now());
4834            if verify_inconclusive(&last.e2e) {
4835                // Still not a real result: `e2e_deferred` is left exactly
4836                // as it was, so `needs_catchup_run` above reads
4837                // `ResourceBlocked` (via `e2e_status`, which checks
4838                // `resource_blocked` before `e2e_deferred`) and retries
4839                // again on the next reentry, rather than recording
4840                // contention as a red e2e and blocking the run on it.
4841                self.state.save()?;
4842                return Ok(());
4843            }
4844            last.e2e_deferred = false;
4845        }
4846        let last = &self.state.reviews[round_idx];
4847        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
4848
4849        match last.e2e_status() {
4850            E2eStatus::Failed => {
4851                let red: Vec<String> = last
4852                    .e2e
4853                    .iter()
4854                    .filter(|o| !o.ok())
4855                    .map(|o| {
4856                        format!(
4857                            "`{}` -> {:?}\n{}",
4858                            o.command,
4859                            o.code,
4860                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4861                        )
4862                    })
4863                    .collect();
4864                self.state
4865                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
4866                self.state.status = RunStatus::Blocked;
4867            }
4868            // `needs_catchup_run` above already retried once this call; if
4869            // it is still blocked, this is magi's own admission it could
4870            // not get a command to run, never a verdict on the patch — the
4871            // run is left exactly where a later reentry can retry again.
4872            E2eStatus::ResourceBlocked => {
4873                self.state.event(
4874                    "review",
4875                    format!(
4876                        "{why}; e2e could not run (shared build cache unavailable); not \
4877                         deciding yet"
4878                    ),
4879                );
4880            }
4881            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
4882                self.state.event(
4883                    "review",
4884                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
4885                );
4886                self.state.status = RunStatus::Gating;
4887            }
4888        }
4889        self.state.save()?;
4890        Ok(())
4891    }
4892
4893    // ----------------------------------------------------------------- gate
4894
4895    async fn gate(&mut self) -> Result<()> {
4896        // Judged by the review record itself, not by `status`: a solo
4897        // candidate's `judge`/`deliberate` skip rewrites `status` on every
4898        // reentry (see `judge`), and trusting it here is exactly how a run
4899        // that exhausted its review budget got gated and merged a second
4900        // time around. `review_conclusion` recomputes the review loop's own
4901        // verdict from the round records themselves — `Gating` for a clean
4902        // round or a hand-off (see `stop_reviewing`), anything else means the
4903        // loop is still going or genuinely blocked.
4904        // A base the winner could not be replayed onto is a decision, not a
4905        // round: there is no landing tree to gate. Read as its own record for
4906        // the same reason the review verdict is.
4907        if self.state.status == RunStatus::Failed
4908            || self
4909                .state
4910                .base_sync
4911                .as_ref()
4912                .is_some_and(|s| s.conflict.is_some())
4913            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4914                != Some(RunStatus::Gating)
4915        {
4916            return Ok(());
4917        }
4918        if self.state.gate_ran {
4919            // `review_loop` derives its conclusion from the clean review
4920            // record on every reentry and therefore puts a completed run back
4921            // in `Gating`. A recorded gate is a stronger, terminal fact:
4922            // retain its original command output (or lack of any, for a repo
4923            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
4924            // doc) and restore `Blocked` on a real failure rather than
4925            // pretending the command is still running or running it a second
4926            // time. `gate_ran == false` remains the only shape — unattempted,
4927            // or a resource-blocked retry — that may still need to execute a
4928            // command.
4929            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
4930                self.state.status = RunStatus::Blocked;
4931                self.state.save()?;
4932            }
4933            return Ok(());
4934        }
4935        let Some(winner) = self.state.winner().cloned() else {
4936            return Ok(());
4937        };
4938        self.state.status = RunStatus::Gating;
4939        let mut outcomes = self.run_gate(&winner).await?;
4940        loop {
4941            // A resource-blocked outcome means the gate command never actually
4942            // ran - the shared build cache could not be acquired or confirmed
4943            // fresh in time - which is evidence about the machine, not about
4944            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
4945            // Recording it as a red gate would mark a run `Blocked` on nothing
4946            // but contention magi has already logged; leaving `self.state.gate`
4947            // empty and `self.state.gate_ran` false instead keeps the shape
4948            // this function already treats as "still needs to run" (see the
4949            // early-return above), so the next call retries the command
4950            // rather than concluding anything.
4951            if verify_inconclusive(&outcomes) {
4952                self.state.save()?;
4953                return Ok(());
4954            }
4955            if outcomes.iter().all(CommandOutcome::ok) {
4956                break;
4957            }
4958            match self.gate_fix_round(&winner, &outcomes).await? {
4959                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
4960                GateFix::Stop => break,
4961                GateFix::Defer => {
4962                    self.state.save()?;
4963                    return Ok(());
4964                }
4965            }
4966        }
4967        let passed = outcomes.iter().all(CommandOutcome::ok);
4968        self.state.gate = outcomes;
4969        self.state.gate_ran = true;
4970        if !passed {
4971            self.state.status = RunStatus::Blocked;
4972            let spent = self.state.gate_fixes.len();
4973            self.state.event(
4974                "gate",
4975                if spent == 0 {
4976                    "gate failed; not merging".to_owned()
4977                } else {
4978                    format!("gate failed after {spent} gate-fix round(s); not merging")
4979                },
4980            );
4981        }
4982        self.state.save()?;
4983        Ok(())
4984    }
4985
4986    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
4987    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
4988    /// after review is clean and never on a candidate awaiting judging.
4989    ///
4990    /// Never fails the run: a non-zero exit or timeout is a warning and a
4991    /// recorded outcome, and the gate remains the single arbiter. Nothing
4992    /// configured means nothing happens - no event, no commit. `commit_all`
4993    /// commits any leftover change under the neutral identity and returns
4994    /// `false` when the tree is clean, so no empty commit is ever made.
4995    async fn run_pre_gate(&mut self, winner: &Candidate) {
4996        let commands = self.state.config.verify.pre_gate.clone();
4997        if commands.is_empty() {
4998            return;
4999        }
5000        let shell = self.state.config.shell();
5001        let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5002        let (outcomes, _) = run_commands(
5003            &mut self.state,
5004            "pre_gate",
5005            "pre_gate",
5006            0,
5007            &shell,
5008            &commands,
5009            &winner.worktree,
5010            timeout,
5011        )
5012        .await;
5013        for o in &outcomes {
5014            if !o.ok() {
5015                tracing::warn!(
5016                    "pre_gate `{}` failed ({:?}); the gate decides",
5017                    o.command,
5018                    o.code
5019                );
5020            }
5021            self.state.event(
5022                "pre_gate",
5023                format!(
5024                    "`{}` -> {}",
5025                    o.command,
5026                    if o.ok() {
5027                        "pass".to_owned()
5028                    } else {
5029                        format!(
5030                            "FAIL ({:?})\n{}",
5031                            o.code,
5032                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5033                        )
5034                    }
5035                ),
5036            );
5037        }
5038        self.state.pre_gate = outcomes;
5039        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5040            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5041                Ok(head) => {
5042                    self.state
5043                        .event("pre_gate", format!("committed mechanical fixes ({head})"));
5044                    self.state.pre_gate_commit = Some(head);
5045                }
5046                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5047            },
5048            Ok(false) => {}
5049            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5050        }
5051        if let Err(e) = self.state.save() {
5052            tracing::warn!("could not persist the pre_gate record: {e:#}");
5053        }
5054    }
5055
5056    /// Run `verify.gate` once against the winner's current tree, logging one
5057    /// event per command. Empty when nothing is configured.
5058    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5059        self.run_pre_gate(winner).await;
5060        let shell = self.state.config.shell();
5061        let gate_commands = self.state.config.verify.gate.clone();
5062        // Zero commands has nothing to run and nothing that could touch the
5063        // shared build cache, so it never needs a lease: `Config::cache_dir`
5064        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
5065        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
5066        // otherwise queue behind an unrelated run's lease and come back
5067        // resource-blocked - `gate_ran` would stay false on nothing but
5068        // cache contention, for a step that had nothing to check in the
5069        // first place.
5070        let outcomes = if gate_commands.is_empty() {
5071            Vec::new()
5072        } else {
5073            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5074            let cache_dir = self.state.config.cache_dir();
5075            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5076            let (outcomes, _) = with_cache_lease(
5077                &mut self.state,
5078                cache_dir.as_deref(),
5079                "gate",
5080                "gate",
5081                &winner.worktree,
5082                &head,
5083                timeout,
5084                "final gate",
5085                |state, budget| {
5086                    let shell = shell.clone();
5087                    let gate_commands = gate_commands.clone();
5088                    let worktree = winner.worktree.clone();
5089                    async move {
5090                        let (outcomes, timed_out_pids) = run_commands(
5091                            state,
5092                            "gate",
5093                            "gate",
5094                            0,
5095                            &shell,
5096                            &gate_commands,
5097                            &worktree,
5098                            budget,
5099                        )
5100                        .await;
5101                        (outcomes, false, timed_out_pids)
5102                    }
5103                },
5104            )
5105            .await;
5106            outcomes
5107        };
5108        if outcomes.is_empty() {
5109            // Nothing configured to check — distinct from every other
5110            // silence in this run's event log, since an empty `gate` alone
5111            // no longer says whether the gate ran at all (see
5112            // `RunState::gate_ran`'s own doc).
5113            self.state.event(
5114                "gate",
5115                "no gate commands configured; nothing to check, passing",
5116            );
5117        }
5118        for o in &outcomes {
5119            self.state.event(
5120                "gate",
5121                format!(
5122                    "`{}` -> {}",
5123                    o.command,
5124                    if o.ok() {
5125                        "pass".to_owned()
5126                    } else {
5127                        format!(
5128                            "FAIL ({:?})\n{}",
5129                            o.code,
5130                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5131                        )
5132                    }
5133                ),
5134            );
5135        }
5136        Ok(outcomes)
5137    }
5138
5139    /// One bounded fix round for a failing gate.
5140    ///
5141    /// The fixer is told the failure came from the gate itself, not from a
5142    /// reviewer, and is shown the failed commands, their exit codes and a tail
5143    /// of their output - whatever `[verify].gate` holds, nothing here knows
5144    /// what those commands run. Only a normal non-zero exit that printed
5145    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
5146    /// command or a full disk says nothing about the code, and a fixer sent
5147    /// after it can only appease the machine. The round is judged by what git
5148    /// says moved, never by the fixer's own report, and `verify.e2e` runs
5149    /// again before the gate does, so a fix cannot trade a green gate for a
5150    /// red e2e unnoticed.
5151    async fn gate_fix_round(
5152        &mut self,
5153        winner: &Candidate,
5154        outcomes: &[CommandOutcome],
5155    ) -> Result<GateFix> {
5156        let cap = self.state.config.graph.gate_fix_rounds;
5157        let spent = self.state.gate_fixes.len();
5158        if spent >= cap {
5159            if cap > 0 {
5160                self.state.event(
5161                    "gate",
5162                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
5163                );
5164            }
5165            return Ok(GateFix::Stop);
5166        }
5167        if !gate_fixable(outcomes) {
5168            self.state.event(
5169                "gate",
5170                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5171                 command or similar); not spending a fix round on it",
5172            );
5173            return Ok(GateFix::Stop);
5174        }
5175        let min_free = self.state.config.disk.min_free_bytes;
5176        if min_free > 0 {
5177            match crate::disk::free_bytes(&winner.worktree) {
5178                Ok(free) if crate::disk::enough_space(free, min_free) => {}
5179                Ok(free) => {
5180                    self.state.event(
5181                        "gate",
5182                        format!(
5183                            "only {free} bytes free ({min_free} required by `[disk] \
5184                             min_free_bytes`); not spending a fix round on a failure the disk \
5185                             may explain"
5186                        ),
5187                    );
5188                    return Ok(GateFix::Stop);
5189                }
5190                Err(e) => {
5191                    self.state.event(
5192                        "gate",
5193                        format!("free disk space could not be measured ({e:#}); no fix round"),
5194                    );
5195                    return Ok(GateFix::Stop);
5196                }
5197            }
5198        }
5199
5200        let attempt = spent + 1;
5201        let run_id = self.state.id.clone();
5202        let prompts = self.state.config.prompts.clone();
5203        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5204        let base = self.landing_base();
5205        let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5206        let seat = self.seat(&fix_seat_key, &fix_spec.id);
5207        let job = SeatJob {
5208            prompt: prompt::gate_fix(
5209                &self.state.instruction,
5210                &failed,
5211                attempt,
5212                cap,
5213                &self.state.config.graph.language,
5214            ),
5215            spec: fix_spec,
5216            seat,
5217            cwd: winner.worktree.clone(),
5218            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5219            allow_write: true,
5220            sessions: self.state.config.graph.sessions,
5221            artifacts: agent::artifacts_dir(&self.state.dir()),
5222            stem: format!("gate-fix-{attempt}"),
5223        };
5224        self.state.event(
5225            "gate",
5226            format!("gate failed; gate-fix round {attempt} of {cap}"),
5227        );
5228        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5229        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5230        let cache = self.state.config.cache_dir();
5231        let ctx = WaveCtx {
5232            run: &run_id,
5233            node: "gate-fix",
5234            prompts: &prompts,
5235            cache: cache.as_deref(),
5236            round: None,
5237        };
5238        let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5239        let mut record = GateFixRecord {
5240            agent: seat.agent.clone(),
5241            failed,
5242            notes: String::new(),
5243            committed: false,
5244            error: None,
5245        };
5246        match out {
5247            AgentOutcome::Ok(o) => {
5248                // A missing report is not a failed fix: the round is judged
5249                // by the tree below, and the report only carries prose.
5250                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5251                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5252                }
5253            }
5254            AgentOutcome::Dropped(_) => {
5255                record.error = Some("the CLI dropped the stream".to_owned());
5256            }
5257            AgentOutcome::Quota(o) => {
5258                self.state.quota.push(QuotaLoss {
5259                    seat: seat.key.clone(),
5260                    node: "gate-fix".to_owned(),
5261                    at: Timestamp::now(),
5262                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5263                });
5264                record.error = Some("rate limited (quota); fixer could not run".to_owned());
5265            }
5266            AgentOutcome::Failed(e) => record.error = Some(e),
5267        }
5268        self.state.seats.insert(seat.key.clone(), seat);
5269        if let Ok(r) = git::rescue_commit(
5270            &winner.worktree,
5271            &format!("magi: gate fix {attempt} (uncommitted work)"),
5272        )
5273        .await
5274        {
5275            self.state.note_withheld("gate-fix", &r.withheld);
5276        }
5277        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5278        record.committed = after != before;
5279        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5280        let note = record.error.clone();
5281        self.state.gate_fixes.push(record);
5282        self.state.save()?;
5283        if !changed {
5284            self.state.event(
5285                "gate",
5286                match note {
5287                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5288                    None => format!("gate-fix round {attempt}: the tree did not change"),
5289                },
5290            );
5291            return Ok(GateFix::Stop);
5292        }
5293        self.state.event(
5294            "gate",
5295            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5296        );
5297
5298        let commands = self.state.config.verify.e2e.clone();
5299        if !commands.is_empty() {
5300            let shell = self.state.config.shell();
5301            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5302            let cache_dir = self.state.config.cache_dir();
5303            let context = format!("gate-fix round {attempt}");
5304            let (e2e, _) = with_cache_lease(
5305                &mut self.state,
5306                cache_dir.as_deref(),
5307                "e2e",
5308                "e2e",
5309                &winner.worktree,
5310                &after,
5311                timeout,
5312                &context,
5313                |state, budget| {
5314                    let shell = shell.clone();
5315                    let commands = commands.clone();
5316                    let context = context.clone();
5317                    let worktree = winner.worktree.clone();
5318                    async move {
5319                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5320                            .await
5321                    }
5322                },
5323            )
5324            .await;
5325            if verify_inconclusive(&e2e) {
5326                return Ok(GateFix::Defer);
5327            }
5328            if e2e.iter().any(|o| !o.ok()) {
5329                self.state.event(
5330                    "gate",
5331                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5332                );
5333                return Ok(GateFix::Stop);
5334            }
5335        }
5336        Ok(GateFix::Retry)
5337    }
5338
5339    // ---------------------------------------------------------------- merge
5340
5341    async fn merge(&mut self) -> Result<()> {
5342        // Same reasoning as `gate`: ask the review and gate records directly
5343        // rather than `status`, which a solo-candidate `judge`/`deliberate`
5344        // skip can rewrite on reentry to something that no longer says
5345        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
5346        // so a hand-off (open findings, green verification) reaches merge
5347        // exactly like a genuinely clean round does.
5348        //
5349        // A run resumed mid-`land` never reaches here at all: `execute`
5350        // recognises `RunStatus::Landing` before it even calls `prep`, and
5351        // routes straight to `run_land` instead. That has to happen a level
5352        // up from this function, not with a check in here, because
5353        // `review_loop`'s own status recomputation (see its doc) runs
5354        // *before* `merge` on every reentry and would otherwise overwrite
5355        // the `Landing` marker with `Gating` before this node ever saw it.
5356        if self
5357            .state
5358            .base_sync
5359            .as_ref()
5360            .is_some_and(|s| s.conflict.is_some())
5361            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5362                != Some(RunStatus::Gating)
5363            // `gate_ran == false` is not "passed" - `gate` leaves it false
5364            // both before it has ever run and when its last attempt was
5365            // resource-blocked (see `Runner::gate`'s own doc), and neither is
5366            // permission to merge on nothing but the review record. Only a
5367            // gate that actually ran - zero commands configured and
5368            // vacuously passed, or one or more that all exited 0 - may
5369            // proceed; `RunState::gate_status` is the single place that
5370            // reading is computed.
5371            || !self.state.gate_status().ok()
5372        {
5373            return Ok(());
5374        }
5375        // This node's own record, not `status`: `status == Ready` is not
5376        // unique to the harmless `MergeMode::None` path this line was
5377        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
5378        // run's PR was closed without merging — and on that run `mode` is
5379        // still `Pr`, so a reentry that fell through here would push and
5380        // open a second pull request. `self.state.merge` is set exactly once
5381        // this node (or `land`) has already produced a verdict, under every
5382        // mode, which is what "already done" actually means here.
5383        if self.state.merge.is_some() {
5384            return Ok(());
5385        }
5386        let Some(winner) = self.state.winner().cloned() else {
5387            return Ok(());
5388        };
5389        let repo = self.state.repo.clone();
5390        let base = self.state.base_branch.clone();
5391        let mode = self.state.config.merge.mode;
5392        let style = self.state.config.merge.style;
5393        let pr = pr_message(&self.state, winner.label);
5394        let message = pr.commit_message();
5395
5396        let outcome = match mode {
5397            MergeMode::None => MergeOutcome {
5398                mode,
5399                ok: true,
5400                detail: manual_merge_command(style, &repo, &winner.branch, &message),
5401            },
5402            MergeMode::Local => {
5403                let on = git::current_branch(&repo).await?;
5404                if on.as_deref() != Some(base.as_str()) {
5405                    MergeOutcome {
5406                        mode,
5407                        ok: false,
5408                        detail: format!(
5409                            "{} has {} checked out, not the base branch {base}",
5410                            repo.display(),
5411                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
5412                        ),
5413                    }
5414                } else if !git::is_clean(&repo).await? {
5415                    MergeOutcome {
5416                        mode,
5417                        ok: false,
5418                        detail: format!("{} is dirty; refusing to merge", repo.display()),
5419                    }
5420                } else {
5421                    let out = match style {
5422                        MergeStyle::Merge => {
5423                            git::merge_no_ff(&repo, &winner.branch, &message).await?
5424                        }
5425                        MergeStyle::Squash => {
5426                            git::merge_squash(&repo, &winner.branch, &message).await?
5427                        }
5428                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5429                    };
5430                    MergeOutcome {
5431                        mode,
5432                        ok: out.ok(),
5433                        detail: if out.ok() { out.stdout } else { out.stderr },
5434                    }
5435                }
5436            }
5437            MergeMode::Pr => {
5438                let remote = self.state.config.merge.remote.clone();
5439                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5440                if !pushed.ok() {
5441                    MergeOutcome {
5442                        mode,
5443                        ok: false,
5444                        detail: pushed.stderr,
5445                    }
5446                } else {
5447                    let out =
5448                        gh_pr_create(&winner.worktree, &base, &winner.branch, &pr.title, &pr.body)
5449                            .await;
5450                    match out {
5451                        Ok(url) => MergeOutcome {
5452                            mode,
5453                            ok: true,
5454                            detail: url,
5455                        },
5456                        Err(e) => MergeOutcome {
5457                            mode,
5458                            ok: false,
5459                            detail: e.to_string(),
5460                        },
5461                    }
5462                }
5463            }
5464        };
5465
5466        self.state.status = match (mode, outcome.ok) {
5467            (MergeMode::None, _) => RunStatus::Ready,
5468            (_, true) => RunStatus::Merged,
5469            (_, false) => RunStatus::Blocked,
5470        };
5471        self.state.event(
5472            "merge",
5473            format!(
5474                "{:?}: {}",
5475                mode,
5476                outcome.detail.lines().next().unwrap_or("")
5477            ),
5478        );
5479        self.state.merge = Some(outcome);
5480        self.state.save()?;
5481
5482        // The PR is open and the run would historically stop here, leaving the
5483        // operator to watch checks, feed review comments back to a fixer, and
5484        // merge. That was done by hand six times in one session before this
5485        // existed. Opt-in, because merging is the one irreversible thing magi
5486        // can do to a repository.
5487        if self.state.config.graph.land
5488            && mode == MergeMode::Pr
5489            && self.state.status == RunStatus::Merged
5490        {
5491            self.run_land().await?;
5492        }
5493        // `run_land` may have left `status` at `Landing` - still waiting on
5494        // CI or the owner's approval, not actually settled - so this has to
5495        // read whatever `status` ended up as here, not the `Merged` this
5496        // function set a few lines up.
5497        self.settle_questions();
5498        Ok(())
5499    }
5500
5501    /// Enter `land`.
5502    ///
5503    /// Shared between a fresh run's first pass through [`Runner::merge`] and
5504    /// a resumed run's re-entry. `land::land` itself is what serialises the
5505    /// two git-mutating moments inside the loop — the rebase push and
5506    /// `gh pr merge` — per repository (see its own doc); nothing here needs
5507    /// to hold a lock across the whole call, and doing so would serialise
5508    /// this run's CI wait against a *different* run's land-approval resume
5509    /// in the same repository, which is exactly the "must not wait on
5510    /// another task" property the daemon's slot-freeing exists to give.
5511    async fn run_land(&mut self) -> Result<()> {
5512        let url = self
5513            .state
5514            .merge
5515            .as_ref()
5516            .map(|m| m.detail.clone())
5517            .unwrap_or_default();
5518        let url = url.lines().next().unwrap_or("").trim().to_owned();
5519        if !url.starts_with("http") {
5520            return Ok(());
5521        }
5522        // A land failure is not a lost run: the work is on a branch and the
5523        // pull request is open, which is exactly where a human takes over.
5524        match land::land(&mut self.state, &url).await {
5525            Ok(pr) if self.state.parked => {
5526                // `land` already saved the parked marker; nothing here
5527                // overrides `status` back to a terminal value while an
5528                // approval is still outstanding.
5529                let _ = pr;
5530            }
5531            Ok(pr) => {
5532                self.state.status = match pr.state {
5533                    land::PrLifecycle::Merged => RunStatus::Merged,
5534                    _ => RunStatus::Blocked,
5535                };
5536                // Downstream of a confirmed merge only - see
5537                // `bump::should_release_bump`'s own doc for why this one
5538                // check covers all three of `land`'s success paths.
5539                // Best-effort: the run already landed, so a failure here
5540                // (the decision call, `gh`, `cargo`) is recorded and never
5541                // turns a landed run into a failed one.
5542                if bump::should_release_bump(self.state.status)
5543                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
5544                {
5545                    // Deliberately only an event: most `Err`s here mean the
5546                    // bump did not apply (no `Cargo.toml`, no agent
5547                    // installed, an unusable decision), not that a release
5548                    // PR is stranded. `after_merge` raises its own notice
5549                    // once a PR exists and needs a human.
5550                    self.state
5551                        .event("bump", format!("release bump skipped: {e:#}"));
5552                }
5553                self.state.save()?;
5554            }
5555            Err(e) => {
5556                self.state.status = RunStatus::Blocked;
5557                self.state.event("land", format!("gave up: {e}"));
5558                self.state.save()?;
5559            }
5560        }
5561        Ok(())
5562    }
5563
5564    // -------------------------------------------------------------- helpers
5565
5566    /// Fetch or create a seat, keeping its conversation across nodes.
5567    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
5568        if let Some(existing) = self.state.seats.get(key)
5569            && existing.agent == agent
5570        {
5571            return existing.clone();
5572        }
5573        let fresh = SeatState::new(key, agent, self.state.seed);
5574        self.state.seats.insert(key.to_owned(), fresh.clone());
5575        fresh
5576    }
5577
5578    /// A candidate rendered for judging, with the leak policy applied.
5579    fn view(&self, c: &Candidate) -> CandidateView {
5580        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
5581            .unwrap_or_default();
5582        let (patch, _) = blind::sanitize_patch(
5583            &format!("candidate {} patch", c.label),
5584            &raw,
5585            &self.state.config.blind,
5586        );
5587        CandidateView {
5588            label: c.label,
5589            branch: c.branch.clone(),
5590            summary: c.summary.clone(),
5591            stat: c.stat.clone(),
5592            patch,
5593        }
5594    }
5595
5596    /// The full candidate set as prompt text, for seats with no live session.
5597    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
5598        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
5599        prompt::judge(
5600            "(see above)",
5601            &views,
5602            self.roles.judges.len(),
5603            base_short,
5604            "en",
5605        )
5606    }
5607
5608    /// Anonymised transcript for judge `self_idx`.
5609    ///
5610    /// The initial rankings are always the opening statements. Seeding them
5611    /// only when no turn had been taken yet meant every judge after the first
5612    /// argued against a single voice instead of against the actual split — the
5613    /// disagreement is the information, so it is always on the table.
5614    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
5615        let mut turns = Vec::new();
5616        for j in &self.state.judgements {
5617            if j.ranking.is_empty() {
5618                continue;
5619            }
5620            let reasons = j
5621                .reasons
5622                .iter()
5623                .map(|(k, v)| format!("- {k}: {v}"))
5624                .collect::<Vec<_>>()
5625                .join("\n");
5626            turns.push(Turn {
5627                who: format!("Judge {} (opening ranking)", j.judge),
5628                is_self: j.judge == self_idx + 1,
5629                body: format!(
5630                    "Ranked {}{}{reasons}",
5631                    j.ranking.iter().collect::<String>(),
5632                    if reasons.is_empty() {
5633                        ""
5634                    } else {
5635                        ", because:\n"
5636                    }
5637                ),
5638            });
5639        }
5640        for t in self
5641            .state
5642            .deliberation
5643            .iter()
5644            .flat_map(|r| r.turns.iter())
5645            .chain(current)
5646        {
5647            turns.push(Turn {
5648                who: format!("Judge {}", t.judge),
5649                is_self: t.judge == self_idx + 1,
5650                body: t.body.clone(),
5651            });
5652        }
5653        turns
5654    }
5655}
5656
5657/// Does this seat still hold the context a follow-up prompt would rely on?
5658fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
5659    agent::has_session(spec.kind, seat, sessions)
5660}
5661
5662/// The next entry in `roster` after `start`, never wrapping back to the
5663/// front, whose id is not in `tried` yet.
5664///
5665/// Starts one past `start` rather than at the front of `roster`: `start` is
5666/// the seat's own original position, and a seat whose candidate slot already
5667/// sits on the roster's second entry must fall through to the third next, not
5668/// restart at the first — which is very likely a different candidate's own
5669/// agent already. Never wraps back past `start`, for the same reason: an
5670/// entry earlier in the roster than the seat's own position is almost
5671/// certainly some *other* candidate slot's own agent, and once the tail of
5672/// the roster is exhausted there are no more untried agents for *this* seat
5673/// to fall through to — the caller's fallback chain ends there, exactly as
5674/// "no further untried agents remain in the list for that seat" asks for.
5675///
5676/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
5677/// the same id twice (an operator's `roles.implementers` typo, or a
5678/// `[[agents]]` list reused across roles) must not let
5679/// [`Runner::resume_quota_losses`] retry that id forever — one forward pass
5680/// over `roster` either finds an untried id or runs out, so this always
5681/// terminates regardless of duplicates.
5682fn next_untried_implementer<'a>(
5683    roster: &'a [AgentSpec],
5684    start: usize,
5685    tried: &BTreeSet<String>,
5686) -> Option<&'a AgentSpec> {
5687    roster
5688        .get(start + 1..)?
5689        .iter()
5690        .find(|s| !tried.contains(&s.id))
5691}
5692
5693/// Did this reply report running a command whose own CLI never confirmed an
5694/// exit status?
5695///
5696/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
5697/// command *finished* (see that type's own doc), so this can only be `true`
5698/// for a command whose completion event carried no readable exit code — not
5699/// for one that simply is not mentioned at all. That is the one signal this
5700/// crate can read, from the same record `state.jobs` renders, about a reply
5701/// standing next to work its own CLI cannot vouch for finishing; it is
5702/// deliberately not a check on the exit code's *value* (a fixer legitimately
5703/// runs a command that fails mid-iteration before it succeeds) and not a
5704/// guess at a command still running in the background (which emits no event
5705/// at all, and so leaves no evidence here to find).
5706fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
5707    commands.iter().any(|c| c.exit_code.is_none())
5708}
5709
5710/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
5711/// trusted as a verified no-op — the adoption guard's own text-level half.
5712///
5713/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
5714/// timed out): a marker only earns the benefit of the doubt from a turn the
5715/// CLI itself vouches for finishing properly, the same house style
5716/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
5717/// *fix* report to for `commands`. A candidate that timed out, exited
5718/// non-zero, or left a command unconfirmed is read as the ordinary loss it
5719/// is, whatever prose it wrote — this returns `None` before it ever looks at
5720/// `text`. The remaining guards (the tree really is empty, the evidence is
5721/// non-empty) are the caller's: this only reads what the reply *claimed*.
5722fn verified_noop_claim(
5723    usable: bool,
5724    commands: &[agent::CommandEvidence],
5725    text: &str,
5726) -> Option<String> {
5727    (usable && !has_unconfirmed_command(commands))
5728        .then(|| verdict::verified_noop(text))
5729        .flatten()
5730}
5731
5732fn short(commit: &str) -> String {
5733    commit.chars().take(7).collect()
5734}
5735
5736fn make_executable(path: &Path) -> Result<()> {
5737    #[cfg(unix)]
5738    {
5739        use std::os::unix::fs::PermissionsExt as _;
5740        let mut perms = std::fs::metadata(path)?.permissions();
5741        perms.set_mode(0o755);
5742        std::fs::set_permissions(path, perms)?;
5743    }
5744    #[cfg(not(unix))]
5745    {
5746        let _ = path;
5747    }
5748    Ok(())
5749}
5750
5751/// What every seat in one batch shares: where the answers are attributed, the
5752/// prompt overlay they inherit, and the build cache they are told to use.
5753///
5754/// A struct rather than four more parameters: `wave` also needs the run's
5755/// state (to record who is answering right now) and the attempt number, and
5756/// eight positional arguments is both unreadable and a clippy error.
5757struct WaveCtx<'a> {
5758    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
5759    /// paid for it.
5760    run: &'a str,
5761    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
5762    node: &'a str,
5763    prompts: &'a Prompts,
5764    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
5765    cache: Option<&'a Path>,
5766    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
5767    /// `JobRecord::round`. `None` for every other node.
5768    round: Option<usize>,
5769}
5770
5771/// Run one job, honouring the parallelism budget.
5772async fn run_one(
5773    job: SeatJob,
5774    sem: Arc<Semaphore>,
5775    ctx: &WaveCtx<'_>,
5776    state: &mut RunState,
5777    attempt: usize,
5778) -> (SeatState, AgentOutcome) {
5779    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
5780        .await
5781        .pop()
5782        .expect("one job in, one result out");
5783    (seat, out)
5784}
5785
5786/// Run every job concurrently, capped by the semaphore, preserving order.
5787///
5788/// Every seat in the batch is recorded into [`RunState::active`] before the
5789/// wave starts and cleared as each answer lands, so the run's own record says
5790/// who is still being waited on rather than only who finished.
5791async fn wave(
5792    jobs: Vec<SeatJob>,
5793    sem: Arc<Semaphore>,
5794    ctx: &WaveCtx<'_>,
5795    state: &mut RunState,
5796    attempt: usize,
5797) -> Vec<(usize, SeatState, AgentOutcome)> {
5798    let WaveCtx {
5799        run,
5800        node,
5801        prompts,
5802        cache,
5803        round,
5804    } = *ctx;
5805    for job in &jobs {
5806        state.seat_started(node, &job.seat.key, job.timeout, attempt);
5807    }
5808    if let Err(e) = state.save() {
5809        // A failed persist of "who is answering right now" must not abort the
5810        // wave: the seats are already being asked, and the alternative is
5811        // losing the answers to save a status line nobody may even be
5812        // watching.
5813        tracing::warn!("could not persist in-progress seats: {e:#}");
5814    }
5815    // Hold the shared build cache's lease for the whole batch, not per job:
5816    // several candidates (an implement wave) or a fixer legitimately share
5817    // one cache concurrently within this run, and that stays untouched — a
5818    // single lease taken once for the whole wave and released once it is
5819    // done is what stops a *different* borrower (another run's own wave, its
5820    // e2e/gate, a human's `magi review`) from interleaving a build into the
5821    // same directory while this one is in flight. Best-effort, not
5822    // all-or-nothing: a wave that cannot get the lease within its own
5823    // longest job's budget still runs — an hour of paid implementer calls is
5824    // not thrown away over cache contention — but every write-allowed seat
5825    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
5826    // below), the same fallback a read-only seat always gets, rather than
5827    // building into a directory this run was never granted. The identity
5828    // record is still invalidated below either way, so the next tracked
5829    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
5830    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
5831    let wait_started = Instant::now();
5832    let cache_guard = if let Some(cache_dir) = cache {
5833        if jobs_had_a_writer {
5834            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
5835            let budget = jobs
5836                .iter()
5837                .map(|j| j.timeout)
5838                .max()
5839                .unwrap_or(Duration::from_secs(60));
5840            acquire_cache_lease(state, cache_dir, &owner, budget, node)
5841                .await
5842                .ok()
5843        } else {
5844            None
5845        }
5846    } else {
5847        None
5848    };
5849    // Carved out of each job's own budget, not added on top of it: a seat
5850    // that waited behind the lease must not also get its full timeout
5851    // afterward, or a run contended on the cache could double the time it
5852    // spends per wave. `saturating_sub` floors at zero rather than
5853    // wrapping - a job whose whole budget was spent waiting starts with
5854    // none left, which is the honest number, not a free minimum.
5855    let waited_for_lease = wait_started.elapsed();
5856    let mut set = tokio::task::JoinSet::new();
5857    let overlay = prompts.overlay(node);
5858    for (i, mut job) in jobs.into_iter().enumerate() {
5859        job.timeout = job.timeout.saturating_sub(waited_for_lease);
5860        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
5861        if cache.is_some() {
5862            job.prompt.push('\n');
5863            job.prompt
5864                .push_str(&prompt::build_cache_note(node, job.allow_write));
5865        }
5866        let sem = Arc::clone(&sem);
5867        let run = run.to_owned();
5868        let node = node.to_owned();
5869        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
5870        // `prompt::build_cache_note`'s doc for why setting it anyway is
5871        // exactly how a sandboxed reviewer's write refusal got reported as a
5872        // defect in the patch, not a property of its own seat. And a
5873        // write-allowed one is handed it only when the lease above was
5874        // actually acquired: a wave that could not get it (`cache_guard` is
5875        // `None`, see its own comment) must not send seats to build into a
5876        // directory this run does not hold - that is the exact concurrent,
5877        // unmanaged-write race this module exists to prevent, not something
5878        // "proceeding anyway" is allowed to reintroduce.
5879        let cache = cache
5880            .filter(|_| job.allow_write && cache_guard.is_some())
5881            .map(Path::to_path_buf);
5882        set.spawn(async move {
5883            let _permit = sem.acquire().await;
5884            let mut seat = job.seat;
5885            let out = agent::invoke(
5886                &job.spec,
5887                &mut seat,
5888                &Invocation {
5889                    cwd: &job.cwd,
5890                    prompt: &job.prompt,
5891                    timeout: job.timeout,
5892                    allow_write: job.allow_write,
5893                    sessions: job.sessions,
5894                    artifacts: &job.artifacts,
5895                    stem: &job.stem,
5896                    run: &run,
5897                    node: &node,
5898                    cache_dir: cache.as_deref(),
5899                    attachments: &[],
5900                },
5901            )
5902            .await;
5903            let out = match out {
5904                Ok(o) if o.usable() => AgentOutcome::Ok(o),
5905                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
5906                // Billed work the CLI failed to hand over is not an ordinary
5907                // failure, but its text is the CLI's raw error JSON, not an
5908                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
5909                // read it as one by forgetting to check. `usable()` is always
5910                // false here (dropped implies an empty response), so this has
5911                // to be checked before the catch-all `Failed` below or the
5912                // one shape this exists for is lost with the rest.
5913                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
5914                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
5915                Ok(o) => AgentOutcome::Failed(format!(
5916                    "exited with {:?} and no usable output",
5917                    o.exit_code
5918                )),
5919                Err(e) => AgentOutcome::Failed(e.to_string()),
5920            };
5921            (i, seat, out)
5922        });
5923    }
5924    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
5925    while let Some(joined) = set.join_next().await {
5926        let (i, seat, out) = match joined {
5927            Ok(v) => v,
5928            // No seat to clear: a panicked task never reported which one it
5929            // was. The defensive sweep below this loop is what stops that
5930            // seat's `active` entry from surviving forever.
5931            Err(e) => {
5932                tracing::error!("agent task panicked: {e}");
5933                continue;
5934            }
5935        };
5936        state.seat_finished(&seat.key);
5937        record_jobs(state, node, round, &seat.key, &out);
5938        if let Err(e) = state.save() {
5939            tracing::warn!("could not persist a seat's completion: {e:#}");
5940        }
5941        if collected.len() <= i {
5942            collected.resize_with(i + 1, || None);
5943        }
5944        collected[i] = Some((i, seat, out));
5945    }
5946    // Belt-and-braces for the panic branch above: every seat this exact batch
5947    // started shares this `(node, attempt)` pair, and every seat that finished
5948    // normally already cleared itself, so anything left tagged with it here
5949    // can only be a panicked task's leftover. Cleared unconditionally rather
5950    // than left to read as still answering forever.
5951    if state
5952        .active
5953        .values()
5954        .any(|a| a.node == node && a.attempt == attempt)
5955    {
5956        state
5957            .active
5958            .retain(|_, a| !(a.node == node && a.attempt == attempt));
5959        if let Err(e) = state.save() {
5960            tracing::warn!("could not persist the end of a wave: {e:#}");
5961        }
5962    }
5963    // Whether or not the lease above was actually held, several worktrees
5964    // may just have built into the cache with nothing here able to name one
5965    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
5966    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
5967    // clean it might not have strictly needed; trusting a stale match would
5968    // cost it a wrong answer.
5969    if let Some(cache_dir) = cache
5970        && jobs_had_a_writer
5971    {
5972        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
5973    }
5974    if let Some(guard) = cache_guard {
5975        guard.release();
5976    }
5977    collected.into_iter().flatten().collect()
5978}
5979
5980/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
5981/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
5982/// this is data collection, not the fix-specific completion contract in
5983/// [`Runner::continue_fix_report`], and applies regardless of which node
5984/// asked.
5985///
5986/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
5987/// evidence from; `Failed` does not, and correctly contributes nothing — a
5988/// timeout or crash is not itself evidence about a command the seat may have
5989/// started.
5990fn record_jobs(
5991    state: &mut RunState,
5992    node: &str,
5993    round: Option<usize>,
5994    seat: &str,
5995    out: &AgentOutcome,
5996) {
5997    let commands: &[agent::CommandEvidence] = match out {
5998        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
5999        AgentOutcome::Failed(_) => &[],
6000    };
6001    let checked_at = Timestamp::now();
6002    for c in commands {
6003        state.jobs.push(JobRecord {
6004            node: node.to_owned(),
6005            round,
6006            seat: seat.to_owned(),
6007            id: c.id.clone(),
6008            description: c.description.clone(),
6009            checked_at,
6010            status: match c.exit_code {
6011                Some(0) => JobStatus::Completed,
6012                Some(_) => JobStatus::Failed,
6013                None => JobStatus::Unknown,
6014            },
6015            exit_code: c.exit_code,
6016            result_summary: c.result_summary.clone(),
6017            source: c.source.clone(),
6018        });
6019    }
6020}
6021
6022/// Is a review round clean, given how many reviewer seats answered against
6023/// how many the round expected?
6024///
6025/// A seat that never answered (timeout, crash, unparsable output) is not a
6026/// seat that read the patch and found nothing — treating it as such is
6027/// exactly the bug this function exists to close. Under the default `block`
6028/// policy a missing seat can never be clean; `warn` still requires the seats
6029/// that *did* answer to have found nothing blocking and verification to be
6030/// green.
6031///
6032/// `quota_missing` narrows that `block` default for exactly one cause of
6033/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
6034/// a session limit lifts by the very next round buys nothing — the seat is
6035/// asked again with the same quota — so once every missing seat is accounted
6036/// for by a quota loss (and at least one seat *did* answer, so a decision has
6037/// something to rest on) the round is decided on the panel that could answer,
6038/// same as `warn` would. A panel that lost every seat to quota is not
6039/// decided here: `answered == 0` falls through to the existing `block`
6040/// fallback so a fully collapsed panel still waits rather than landing on no
6041/// review at all.
6042fn round_is_clean(
6043    blocking: usize,
6044    e2e_ok: bool,
6045    answered: usize,
6046    expected: usize,
6047    quota_missing: usize,
6048    policy: IncompleteReviewPolicy,
6049) -> bool {
6050    if blocking != 0 || !e2e_ok {
6051        return false;
6052    }
6053    if answered == expected || policy == IncompleteReviewPolicy::Warn {
6054        return true;
6055    }
6056    answered > 0 && expected - answered <= quota_missing
6057}
6058
6059/// The review loop's own conclusion, derived entirely from its persisted
6060/// round records and the round budget that produced them — never from
6061/// `status`, so a reentry (or `gate`/`merge` reading it independently)
6062/// recomputes the identical answer regardless of what an earlier node in the
6063/// same walk, or a previous walk, did to `status`.
6064///
6065/// `None` while more rounds remain to try, including when review never ran
6066/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
6067/// gone clean, or the budget is spent, or the tree has stopped moving (see
6068/// [`STAGNANT_LIMIT`]), the answer is one of two things:
6069///
6070/// - An incomplete panel that raised nothing is missing input, not a
6071///   verified tree — never a hand-off candidate, whatever verification said
6072///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
6073/// - Otherwise, green e2e on the last round hands off (see
6074///   [`Runner::stop_reviewing`]); red e2e blocks.
6075///
6076/// A last round whose own verification is still `ResourceBlocked` — magi
6077/// itself never got a command to run, not evidence the patch is broken —
6078/// is neither: this returns `None` for it too, the same as "more rounds
6079/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
6080/// handling of that shape) instead of this cheap recomputation guessing a
6081/// verdict a real attempt never produced.
6082fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
6083    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
6084        return Some(RunStatus::Gating);
6085    }
6086    let last = reviews.last()?;
6087    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
6088    if reviews.len() < max_rounds && !stagnant {
6089        return None;
6090    }
6091    if last.incomplete() && last.blocking == 0 {
6092        return Some(RunStatus::Blocked);
6093    }
6094    if last.e2e_status() == E2eStatus::ResourceBlocked {
6095        return None;
6096    }
6097    Some(if last.e2e.iter().all(CommandOutcome::ok) {
6098        RunStatus::Gating
6099    } else {
6100        RunStatus::Blocked
6101    })
6102}
6103
6104/// How long a re-ask may take, given the budget the first attempt had.
6105///
6106/// A `nudged` retry is a request to restate an answer the seat has already
6107/// worked out: it carries no new work, so it does not deserve the original
6108/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
6109/// 133 seconds while a third sat for over ten minutes on a resumed session
6110/// holding 410 KB of prior output - and because the retry had inherited the
6111/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
6112/// judging round whose other seats were long finished.
6113///
6114/// A quarter of the budget, with a floor so that a deliberately short timeout
6115/// does not collapse to nothing. A retry that re-sends the whole prompt
6116/// (because the seat kept no context) is the original job again, and keeps the
6117/// original budget.
6118fn retry_budget(full: Duration, nudged: bool) -> Duration {
6119    if nudged {
6120        (full / 4).max(Duration::from_secs(120)).min(full)
6121    } else {
6122        full
6123    }
6124}
6125
6126/// Run a wave and parse each reply, re-asking the seats whose reply was
6127/// unusable.
6128///
6129/// The re-ask is a nudge rather than the whole prompt again when the seat still
6130/// holds its conversation, which is the difference between a cheap retry and
6131/// paying for the entire candidate set twice.
6132///
6133/// A seat that hits a rate limit is **not** re-asked: the same call will fail
6134/// the same way until the limit resets, so spending a retry attempt on it is
6135/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
6136/// like any other absent seat — the caller decides whether the panel still has
6137/// a quorum.
6138#[allow(clippy::too_many_arguments)]
6139async fn ask_json_wave<T>(
6140    jobs: Vec<SeatJob>,
6141    sem: Arc<Semaphore>,
6142    retries: usize,
6143    ctx: &WaveCtx<'_>,
6144    losses: &mut Vec<QuotaLoss>,
6145    state: &mut RunState,
6146    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
6147) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6148where
6149    T: serde::de::DeserializeOwned + Send + 'static,
6150{
6151    let n = jobs.len();
6152    let originals: Vec<SeatJob> = jobs;
6153    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
6154    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
6155    // Which attempt each seat's `done[i]` reflects — 0 for a first-ask
6156    // answer, N once it has gone through N nudges. Read back once this
6157    // returns, so a caller building a history record (`ReviewRecord`) can
6158    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
6159    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
6160    // field's own doc.
6161    let mut attempts_used: Vec<usize> = vec![0; n];
6162    let mut pending: Vec<usize> = (0..n).collect();
6163
6164    for attempt in 0..=retries {
6165        if pending.is_empty() {
6166            break;
6167        }
6168        let mut batch = Vec::with_capacity(pending.len());
6169        for &i in &pending {
6170            let src = &originals[i];
6171            // The prompt and the budget are one decision: a nudge restates
6172            // finished work, a re-sent prompt redoes it.
6173            let (prompt, timeout) = if attempt == 0 {
6174                (src.prompt.clone(), src.timeout)
6175            } else {
6176                let why = done[i]
6177                    .as_ref()
6178                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
6179                    .unwrap_or_else(|| "no parsable answer".to_owned());
6180                let nudge = prompt::nudge(&why);
6181                let nudged = has_context(&src.spec, &seats[i], src.sessions);
6182                let prompt = if nudged {
6183                    nudge
6184                } else {
6185                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
6186                };
6187                (prompt, retry_budget(src.timeout, nudged))
6188            };
6189            batch.push(SeatJob {
6190                spec: src.spec.clone(),
6191                seat: seats[i].clone(),
6192                cwd: src.cwd.clone(),
6193                prompt,
6194                timeout,
6195                allow_write: src.allow_write,
6196                sessions: src.sessions,
6197                artifacts: src.artifacts.clone(),
6198                stem: if attempt == 0 {
6199                    src.stem.clone()
6200                } else {
6201                    format!("{}-retry{attempt}", src.stem)
6202                },
6203            });
6204        }
6205
6206        if attempt > 0 {
6207            let seats_out: Vec<&str> = pending
6208                .iter()
6209                .map(|&i| originals[i].seat.key.as_str())
6210                .collect();
6211            state.event(
6212                ctx.node,
6213                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
6214            );
6215        }
6216        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
6217        let mut still = Vec::new();
6218        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
6219            seats[i] = seat;
6220            let (parsed, quota) = match out {
6221                AgentOutcome::Ok(o) => (
6222                    match verdict::extract_json::<T>(&o.text) {
6223                        Ok(v) => match validate(&v) {
6224                            Ok(()) => Ok((v, o)),
6225                            Err(e) => Err(e),
6226                        },
6227                        Err(e) => Err(e),
6228                    },
6229                    false,
6230                ),
6231                AgentOutcome::Quota(o) => {
6232                    losses.push(QuotaLoss {
6233                        seat: originals[i].seat.key.clone(),
6234                        node: ctx.node.to_owned(),
6235                        at: Timestamp::now(),
6236                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6237                    });
6238                    (
6239                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
6240                        true,
6241                    )
6242                }
6243                // Not a parseable answer, but also not worth a special-cased
6244                // retry here: the nudge loop above already re-asks anything
6245                // that fails to parse, which is exactly what a dropped stream
6246                // needs. Just don't hand its raw error JSON to `extract_json`.
6247                AgentOutcome::Dropped(o) => {
6248                    let why = o
6249                        .dropped
6250                        .as_ref()
6251                        .map(|d| d.why.as_str())
6252                        .unwrap_or("the CLI ended the stream without delivering its answer");
6253                    (
6254                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
6255                        false,
6256                    )
6257                }
6258                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
6259            };
6260            let failed = parsed.is_err();
6261            done[i] = Some(parsed);
6262            attempts_used[i] = attempt;
6263            // Do not re-ask a rate-limited seat (quota) — a retry is known to
6264            // fail the same way; and never re-ask a seat that already parsed.
6265            if failed && !quota {
6266                still.push(i);
6267            }
6268        }
6269        pending = still;
6270    }
6271
6272    seats
6273        .into_iter()
6274        .zip(done)
6275        .zip(attempts_used)
6276        .map(|((seat, res), attempts)| {
6277            (
6278                seat,
6279                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
6280                attempts,
6281            )
6282        })
6283        .collect()
6284}
6285
6286/// Acquire the shared build cache's lease, waiting out contention within
6287/// `budget` (never past it — see AGENTS.md's build-cache section on why an
6288/// unbounded wait is never acceptable).
6289///
6290/// A first, non-blocking check happens before ever waiting; if it finds the
6291/// lease busy, that fact is logged as a `verify` event *and* flushed with
6292/// [`RunState::save`] immediately — not only once the wait finally succeeds
6293/// or gives up — so a `magi show` run by a different process while this one
6294/// is still waiting reads a `run.json` that says so, rather than whatever it
6295/// looked like before the wait started. The same applies to the terminal
6296/// failure: logged and saved before this returns `Err`, so a caller that
6297/// could not get the lease at all still leaves a legible record of why.
6298async fn acquire_cache_lease(
6299    state: &mut RunState,
6300    cache_dir: &Path,
6301    owner: &crate::cache::Owner,
6302    budget: Duration,
6303    context: &str,
6304) -> Result<crate::cache::Guard> {
6305    let home = crate::run::home();
6306    let started = Instant::now();
6307    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
6308        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
6309        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
6310        Err(e) => {
6311            state.event(
6312                "verify",
6313                format!("{context}: could not check the shared build cache: {e:#}"),
6314            );
6315            if let Err(e2) = state.save() {
6316                tracing::warn!("could not persist a cache-check failure: {e2:#}");
6317            }
6318            return Err(e);
6319        }
6320    };
6321    state.event(
6322        "verify",
6323        format!(
6324            "{context}: waiting for the shared build cache at {} ({})",
6325            cache_dir.display(),
6326            busy.describe()
6327        ),
6328    );
6329    if let Err(e) = state.save() {
6330        tracing::warn!("could not persist a cache wait: {e:#}");
6331    }
6332    let remaining = budget.saturating_sub(started.elapsed());
6333    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
6334        Ok(g) => Ok(g),
6335        Err(e) => {
6336            state.event("verify", format!("{context}: {e:#}"));
6337            if let Err(e2) = state.save() {
6338                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
6339            }
6340            Err(e)
6341        }
6342    }
6343}
6344
6345/// Run `body` — a verify command batch — while holding the shared build
6346/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
6347/// never interleave with another borrower's build against the same
6348/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
6349/// timeout, or a human's own `magi review`. See the `cache` module doc for
6350/// why this matters more than Cargo's own per-target locking covers — two
6351/// *different* worktrees building the same package name/version into one
6352/// cache directory is a staleness bug, not a lock contention one.
6353///
6354/// The wait for the lease is carved out of `budget`, never on top of it —
6355/// `body` is handed whatever is left, so a caller's own node timeout is the
6356/// only clock involved, exactly what AGENTS.md's build-cache section asks
6357/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
6358/// cache configured at all — this is a pass-through: `body` runs with the
6359/// full budget and nothing is leased.
6360///
6361/// A lease that cannot be acquired within `budget` is reported as a single
6362/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
6363/// verification — the same shape a spawn failure already takes in
6364/// [`run_commands`], so a caller need not special-case it.
6365#[allow(clippy::too_many_arguments)]
6366async fn with_cache_lease<'s, F, Fut>(
6367    state: &'s mut RunState,
6368    cache_dir: Option<&Path>,
6369    node: &str,
6370    seat: &str,
6371    worktree: &Path,
6372    head: &str,
6373    budget: Duration,
6374    context: &str,
6375    body: F,
6376) -> (Vec<CommandOutcome>, bool)
6377where
6378    F: FnOnce(&'s mut RunState, Duration) -> Fut,
6379    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
6380{
6381    let Some(cache_dir) = cache_dir else {
6382        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
6383        return (outcomes, retried);
6384    };
6385    let home = crate::run::home();
6386    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
6387    let started = Instant::now();
6388    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
6389        Ok(g) => g,
6390        Err(e) => {
6391            return (
6392                vec![CommandOutcome {
6393                    command: "(waiting for the shared build cache)".to_owned(),
6394                    code: None,
6395                    output_tail: e.to_string(),
6396                    duration_ms: started.elapsed().as_millis() as u64,
6397                    resource_blocked: true,
6398                }],
6399                false,
6400            );
6401        }
6402    };
6403    let identity = crate::cache::Identity::new(worktree, head);
6404    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
6405        // A failed freshness check means this process cannot vouch for what
6406        // is sitting in the cache right now - on Windows this is exactly the
6407        // "a stale test executable is still locked, `cargo clean -p` cannot
6408        // remove it" case the evidence log records. Running verify anyway
6409        // and reporting whatever it says would let a result nobody can trust
6410        // stand for the tree it claims to have checked; fail the step
6411        // instead of the patch.
6412        state.event(
6413            "verify",
6414            format!(
6415                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
6416                worktree.display(),
6417                short(head)
6418            ),
6419        );
6420        guard.release();
6421        return (
6422            vec![CommandOutcome {
6423                command: "(confirming the shared build cache is fresh)".to_owned(),
6424                code: None,
6425                output_tail: e.to_string(),
6426                duration_ms: started.elapsed().as_millis() as u64,
6427                resource_blocked: true,
6428            }],
6429            false,
6430        );
6431    }
6432    let remaining = budget.saturating_sub(started.elapsed());
6433    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
6434    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
6435    // `start_kill`); confirm it actually has before handing the directory to
6436    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
6437    // for what this can and cannot see.
6438    if !timed_out_pids.is_empty() {
6439        wait_for_timed_out_children_to_die(&timed_out_pids).await;
6440    }
6441    guard.release();
6442    (outcomes, retried)
6443}
6444
6445/// Poll `pids` — commands [`run_commands`] reports as still running when its
6446/// own timeout elapsed — until every one is confirmed gone, or
6447/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
6448///
6449/// Real confirmation where confirmation is possible, not a substitute for
6450/// full process-tree observation: a grandchild the timed-out process spawned
6451/// and that survives independently of it is invisible to a pid check the
6452/// same way it always was, and continuing to observe and collect *that*
6453/// stays a different piece of work with its own owner. This only narrows a
6454/// fixed blind wait into an actual check of the pids this process does know
6455/// about.
6456async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
6457    wait_for_pids_with(
6458        pids,
6459        crate::proc::pid_alive,
6460        LEASE_RELEASE_POLL,
6461        LEASE_RELEASE_MAX_WAIT,
6462    )
6463    .await;
6464}
6465
6466/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
6467/// interval and ceiling supplied by the caller, so the polling *logic* -
6468/// returns as soon as every pid reports dead, gives up at the ceiling
6469/// otherwise - is testable on millisecond durations without asking the real
6470/// OS about a pid at all.
6471async fn wait_for_pids_with<F: Fn(u32) -> bool>(
6472    pids: &[u32],
6473    alive: F,
6474    poll: Duration,
6475    max_wait: Duration,
6476) {
6477    let deadline = Instant::now() + max_wait;
6478    loop {
6479        if pids.iter().all(|&pid| !alive(pid)) {
6480            return;
6481        }
6482        if Instant::now() >= deadline {
6483            return;
6484        }
6485        tokio::time::sleep(poll).await;
6486    }
6487}
6488
6489/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
6490/// admission that it could not even get a verify command to run, as opposed
6491/// to evidence the command actually produced? A caller that would otherwise
6492/// read a resource-blocked outcome as a red command must check this first:
6493/// see [`Runner::gate`], which retries rather than records `Blocked` when
6494/// this is true.
6495fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
6496    outcomes.iter().any(|o| o.resource_blocked)
6497}
6498
6499/// What [`Runner::gate_fix_round`] decided.
6500enum GateFix {
6501    /// The tree changed and `verify.e2e` is still green: run the gate again.
6502    Retry,
6503    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
6504    /// last failure stands and the run ends blocked.
6505    Stop,
6506    /// `verify.e2e` could not run after the fix (magi's own contention):
6507    /// decide nothing now, a later reentry retries.
6508    Defer,
6509}
6510
6511/// Is every red command in `outcomes` an ordinary failure the code could
6512/// explain: it ran, exited non-zero, and said something?
6513///
6514/// A timeout, a spawn failure and a killed process all leave `code` `None`;
6515/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
6516/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
6517/// what the command is stays the gate's business.
6518fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
6519    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
6520    red.peek().is_some()
6521        && red.all(|o| {
6522            !o.resource_blocked
6523                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
6524                && !o.output_tail.trim().is_empty()
6525        })
6526}
6527
6528/// Describe one verify command's outcome for the event log, distinguishing a
6529/// build/link failure — the toolchain never produced a binary to run — from
6530/// an actual test failure, since only the latter is a verdict on the patch.
6531fn e2e_outcome_label(o: &CommandOutcome) -> String {
6532    if o.ok() {
6533        return "pass".to_owned();
6534    }
6535    let reason = if o.build_failed() {
6536        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
6537    } else {
6538        format!("FAIL ({:?})", o.code)
6539    };
6540    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
6541}
6542
6543/// Run `verify.e2e`, retrying once if the first attempt could not build or
6544/// link — a build/link failure is frequently a race against a shared
6545/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
6546/// `verify` event per command, tagged with `context` (normally `"round N"`)
6547/// so the two call sites that need this — the ordinary per-round leg in
6548/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
6549/// it will ever call a round green — read identically in the event log.
6550async fn run_e2e_with_retry(
6551    state: &mut RunState,
6552    shell: &[String],
6553    commands: &[String],
6554    worktree: &Path,
6555    timeout: Duration,
6556    context: &str,
6557) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
6558    let (mut e2e, mut timed_out_pids) = run_commands(
6559        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
6560    )
6561    .await;
6562    for o in &e2e {
6563        state.event(
6564            "verify",
6565            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
6566        );
6567    }
6568    // A build/link failure is not a verdict on the patch — it is frequently a
6569    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
6570    // one retry before letting a red like that decide the round.
6571    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
6572    if verify_retried {
6573        state.event(
6574            "verify",
6575            format!(
6576                "{context}: verify could not build/link, not a test result — retrying once \
6577                 before concluding"
6578            ),
6579        );
6580        let retried = run_commands(
6581            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
6582        )
6583        .await;
6584        e2e = retried.0;
6585        // Both attempts' timeouts matter, not just the last one: the first
6586        // attempt's descendants may still be alive alongside the retry's.
6587        timed_out_pids.extend(retried.1);
6588        for o in &e2e {
6589            state.event(
6590                "verify",
6591                format!(
6592                    "{context}: retry `{}` -> {}",
6593                    o.command,
6594                    e2e_outcome_label(o)
6595                ),
6596            );
6597        }
6598    }
6599    (e2e, verify_retried, timed_out_pids)
6600}
6601
6602/// Run configured shell commands in `cwd`, in order. The second element is
6603/// the pid of every command that hit `timeout` and was still running when
6604/// this stopped waiting on it (best-effort: `None` when the platform did not
6605/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
6606/// that releases a shared resource afterward needs to know.
6607///
6608/// Records `task` into [`RunState::active`] at every command boundary
6609/// (`RunState::task_command`) and clears it once the whole list has run
6610/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
6611/// for minutes with no seat and no output of its own to show for it (see
6612/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
6613/// apart from "not yet run" without this), and this is the only place that
6614/// knows which command is running right now and how many are left. Three
6615/// saves per command — start, not per second — matching the same "only at a
6616/// boundary" rule [`wave`] already follows for seats.
6617#[allow(clippy::too_many_arguments)]
6618async fn run_commands(
6619    state: &mut RunState,
6620    node: &str,
6621    task: &str,
6622    attempt: usize,
6623    shell: &[String],
6624    commands: &[String],
6625    cwd: &Path,
6626    timeout: Duration,
6627) -> (Vec<CommandOutcome>, Vec<u32>) {
6628    if commands.is_empty() {
6629        // Nothing to mark as running and nothing to clear — an empty list
6630        // means "not configured", and touching `active` (or the disk) over
6631        // that would be a write for every round of a repo with no
6632        // `verify.e2e` / `verify.gate` commands at all.
6633        return (Vec::new(), Vec::new());
6634    }
6635    let mut out = Vec::new();
6636    let mut timed_out_pids = Vec::new();
6637    let total = commands.len();
6638    for (idx, command) in commands.iter().enumerate() {
6639        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
6640        if let Err(e) = state.save() {
6641            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
6642        }
6643        let started = Instant::now();
6644        let mut cmd = tokio::process::Command::new(&shell[0]);
6645        cmd.quiet();
6646        cmd.args(&shell[1..])
6647            .arg(command)
6648            .current_dir(cwd)
6649            .stdin(std::process::Stdio::null())
6650            .stdout(std::process::Stdio::piped())
6651            .stderr(std::process::Stdio::piped())
6652            .kill_on_drop(true);
6653        let spawned = cmd.spawn();
6654        let (code, body) = match spawned {
6655            Ok(child) => {
6656                // Captured before the child is consumed below: `kill_on_drop`
6657                // only *asks* the process to die when the timeout branch
6658                // drops it, and the pid is the only way anyone downstream can
6659                // later check whether that request actually took.
6660                let pid = child.id();
6661                match tokio::time::timeout(timeout, child.wait_with_output()).await {
6662                    Ok(Ok(o)) => {
6663                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
6664                        body.push_str(&String::from_utf8_lossy(&o.stderr));
6665                        (o.status.code(), body)
6666                    }
6667                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
6668                    Err(_) => {
6669                        if let Some(pid) = pid {
6670                            timed_out_pids.push(pid);
6671                        }
6672                        (None, format!("timed out after {}s", timeout.as_secs()))
6673                    }
6674                }
6675            }
6676            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
6677        };
6678        out.push(CommandOutcome {
6679            command: command.clone(),
6680            code,
6681            output_tail: tail(&body, OUTPUT_TAIL),
6682            duration_ms: started.elapsed().as_millis() as u64,
6683            resource_blocked: false,
6684        });
6685    }
6686    state.task_finished(task);
6687    if let Err(e) = state.save() {
6688        tracing::warn!("could not persist the end of {task}: {e:#}");
6689    }
6690    (out, timed_out_pids)
6691}
6692
6693/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
6694/// section (`report::run`) and in the `merge` event this node records — for
6695/// the operator to run by hand.
6696///
6697/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
6698/// branch whose ruleset forbids merge commits (GitHub's "must not contain
6699/// merge commits", or "require linear history") rejects the push a `--no-ff`
6700/// merge would produce, which is exactly the guidance this function replaces.
6701/// `message`'s first line becomes the squash commit's subject, matching the
6702/// note `report::run` prints alongside this command — see that function for
6703/// why an explicit subject is not optional there.
6704fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
6705    let repo = repo.display();
6706    match style {
6707        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
6708        MergeStyle::Squash => {
6709            // The subject sits inside double quotes, and a title an agent
6710            // wrote may carry the characters that break out of them.
6711            let subject = message
6712                .lines()
6713                .next()
6714                .unwrap_or(branch)
6715                .replace(['\\', '"', '$', '`'], "");
6716            format!(
6717                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
6718            )
6719        }
6720        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
6721    }
6722}
6723
6724/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
6725/// under the hood, rejects a `title` over 256 characters and the whole
6726/// command fails — no PR at all, for a run whose body was otherwise fine
6727/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
6728/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
6729/// always how GitHub counts, plus one character for the trailing ellipsis
6730/// `title_from` may add. It is a margin, not a guarantee — a title packed
6731/// with multi-unit characters could still in principle land close to the
6732/// edge, but a real task title's occasional emoji or accented letter fits
6733/// comfortably inside it.
6734const PR_TITLE_MAX: usize = 240;
6735
6736/// What `merge = "pr"` (and the merge commit of the other modes) says about a
6737/// change: a title and a body describing what was *implemented*, not the task
6738/// that asked for it. A task reads as a request; a reader of the merged
6739/// history wants the change.
6740struct PrMessage {
6741    title: String,
6742    body: String,
6743}
6744
6745impl PrMessage {
6746    /// Title, blank line, body. The first line is the squash/merge commit
6747    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
6748    /// has to stay one sensible line.
6749    fn commit_message(&self) -> String {
6750        format!("{}\n\n{}", self.title, self.body)
6751    }
6752}
6753
6754/// The text after a leading `TITLE:` (any case) on `line`.
6755fn title_marker(line: &str) -> Option<&str> {
6756    let line = line.trim();
6757    let head = line.get(..6)?;
6758    head.eq_ignore_ascii_case("title:")
6759        .then(|| line[6..].trim())
6760}
6761
6762/// The implementer's own one-line title: the `TITLE:` line the implement
6763/// prompt asks for at the top of its SUMMARY. Candidate commits are all
6764/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
6765/// source, and a title that says as much is refused here too.
6766fn summary_title(summary: &str) -> Option<String> {
6767    let first = summary.lines().find(|l| !l.trim().is_empty())?;
6768    let raw = title_marker(first)?;
6769    if raw.is_empty() {
6770        return None;
6771    }
6772    let title = queue::title_from(raw, PR_TITLE_MAX);
6773    let lower = title.to_ascii_lowercase();
6774    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
6775        return None;
6776    }
6777    Some(title)
6778}
6779
6780/// `summary` without its `TITLE:` line, which the pull request title already
6781/// carries.
6782fn summary_without_title(summary: &str) -> String {
6783    let mut lines = summary.trim().lines().peekable();
6784    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
6785        lines.next();
6786    }
6787    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
6788}
6789
6790/// The pull request title and body for the winning candidate.
6791///
6792/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
6793/// the task's own opening line via [`queue::title_from`] when there is none.
6794/// `state.instruction` can open with blank lines (`task_text` only rejects a
6795/// body that is blank *entirely*), which `title_from` skips.
6796///
6797/// Body: the implementer's summary and the fixer's notes, then — when the
6798/// winning review round was not clean — the findings still open and whatever
6799/// the fixer declined, so `merge = "pr"` hands the reader the same material
6800/// `magi show` does. The task follows inside a collapsed block, and the
6801/// footer repeats the run and candidate as plain tags for a reader holding
6802/// only the merged commit or the PR body.
6803fn pr_message(state: &RunState, winner: char) -> PrMessage {
6804    let summary = state
6805        .candidates
6806        .iter()
6807        .find(|c| c.label == winner)
6808        .map(|c| c.summary.as_str())
6809        .unwrap_or_default();
6810    // The fallback is the operator's own words and may not be English; GitHub
6811    // text always is, so a non-English task gets a neutral title instead.
6812    let title = summary_title(summary).unwrap_or_else(|| {
6813        let t = queue::title_from(&state.instruction, PR_TITLE_MAX);
6814        if t.is_ascii() && t.chars().any(|c| c.is_ascii_alphabetic()) {
6815            t
6816        } else {
6817            format!(
6818                "chore: land candidate {} of run {}",
6819                winner.to_ascii_uppercase(),
6820                state.id
6821            )
6822        }
6823    });
6824
6825    let mut body = String::new();
6826    let what = summary_without_title(summary);
6827    if !what.is_empty() {
6828        body.push_str("## Summary\n\n");
6829        body.push_str(&what);
6830        body.push_str("\n\n");
6831    }
6832
6833    let fix = state.reviews.last().and_then(|r| r.fix.as_ref());
6834    if let Some(fix) = fix
6835        && !fix.notes.trim().is_empty()
6836    {
6837        body.push_str("## Review fixes\n\n");
6838        body.push_str(fix.notes.trim());
6839        body.push_str("\n\n");
6840    }
6841
6842    let open = state.open_findings();
6843    if !open.is_empty() {
6844        body.push_str("## Open review findings\n\n");
6845        for f in &open {
6846            body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
6847        }
6848        body.push('\n');
6849    }
6850
6851    if let Some(fix) = fix
6852        && !fix.rejected.is_empty()
6853    {
6854        body.push_str("## Declined by the fixer\n\n");
6855        for r in &fix.rejected {
6856            body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
6857        }
6858        body.push('\n');
6859    }
6860
6861    let task = state.instruction.trim();
6862    let task = if task.is_empty() {
6863        "(empty task)"
6864    } else {
6865        task
6866    };
6867    body.push_str(&format!(
6868        "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
6869        task.replace("</details>", "&lt;/details&gt;")
6870    ));
6871
6872    body.push_str(&format!(
6873        "\n---\nmagi:run/{} magi:candidate-{}\n",
6874        state.id,
6875        winner.to_ascii_lowercase()
6876    ));
6877
6878    // Prompts are advisory; this is the enforced half of the confidentiality
6879    // rule, and it covers the verbatim task in <details> too.
6880    let id = crate::scrub::Identity::current();
6881    PrMessage {
6882        title: crate::scrub::scrub(&title, &id),
6883        body: crate::scrub::scrub(&body, &id),
6884    }
6885}
6886
6887/// `gh pr create`, returning the PR url.
6888async fn gh_pr_create(
6889    cwd: &Path,
6890    base: &str,
6891    head: &str,
6892    title: &str,
6893    body: &str,
6894) -> Result<String> {
6895    let out = tokio::process::Command::new("gh")
6896        .args([
6897            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
6898        ])
6899        .current_dir(cwd)
6900        .quiet()
6901        .stdin(std::process::Stdio::null())
6902        .output()
6903        .await
6904        .context("spawn gh")?;
6905    if out.status.success() {
6906        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
6907    } else {
6908        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
6909    }
6910}
6911
6912/// Tear a run's worktrees and branches down.
6913///
6914/// `home` is where the updated `run.json` is saved (via
6915/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
6916/// a housekeeping pass already has its own honest `home` handed to it, and
6917/// falling through to the global here would write back through whichever
6918/// directory some other process or test pinned into that `OnceLock` first,
6919/// not the one the caller actually resolved its `runs` and `state` from.
6920pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
6921    let repo = state.repo.clone();
6922    let root = state.worktree_root();
6923    let winner = state.tally.as_ref().map(|t| t.winner);
6924    let mut removed = Vec::new();
6925
6926    for i in 0..state.candidates.len() {
6927        let c = state.candidates[i].clone();
6928        let is_winner = Some(c.label) == winner;
6929        if is_winner && !drop_winner {
6930            continue;
6931        }
6932        if c.worktree.exists() {
6933            git::worktree_remove(&repo, &c.worktree).await.ok();
6934            removed.push(c.worktree.to_string_lossy().into_owned());
6935        }
6936        // A branch handed to a later run (and its pull request) is not this
6937        // run's to delete.
6938        let handed_over = state.released_branches.contains(&c.branch);
6939        if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
6940            git::branch_delete(&repo, &c.branch).await.ok();
6941            removed.push(c.branch.clone());
6942        }
6943        state.candidates[i].folded = true;
6944    }
6945
6946    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
6947        let path = name.path();
6948        let keep = !drop_winner
6949            && winner.is_some_and(|w| {
6950                path.file_name()
6951                    .is_some_and(|n| n == format!("cand-{w}").as_str())
6952            });
6953        if keep {
6954            continue;
6955        }
6956        git::worktree_remove(&repo, &path).await.ok();
6957        removed.push(path.to_string_lossy().into_owned());
6958    }
6959
6960    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
6961    // judge worktrees, so once the loop above has cleared all of them out,
6962    // the parent is a bare directory nobody else was ever going to remove -
6963    // git only ever managed what was inside it. Left alone, one of these
6964    // accumulates per fully-folded run; the operator's own machine had 74.
6965    // `remove_if_empty` re-checks rather than assuming: a run whose winner
6966    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
6967    // so does anything a run never claimed that happens to share the bay.
6968    remove_if_empty(&root);
6969
6970    if state.enabled_worktree_config && drop_winner {
6971        // A release, not a raw disable: some sibling run in this repository
6972        // may still hold its own reference (see `git::acquire_worktree_config`),
6973        // and only the last release actually turns the setting back off.
6974        git::release_worktree_config(&repo).await.ok();
6975        state.enabled_worktree_config = false;
6976    }
6977    state.save_under(home)?;
6978    Ok(removed)
6979}
6980
6981/// Remove `dir` if it exists and has nothing in it.
6982///
6983/// Best-effort and silent by design: a directory that is not empty (a run
6984/// whose winner is still parked there, a stray file some other process left)
6985/// is exactly the case this must refuse, and a directory that is already gone
6986/// is not a failure worth reporting either. `std::fs::remove_dir` itself
6987/// already refuses a non-empty directory, so the emptiness check below is
6988/// belt, not suspenders - it is what keeps this from ever attempting the
6989/// removal in the case that matters, rather than trusting `remove_dir`'s
6990/// error path to have no side effects if it ever changed.
6991fn remove_if_empty(dir: &Path) {
6992    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
6993        std::fs::remove_dir(dir).ok();
6994    }
6995}
6996
6997/// Severity of the worst open finding in the last review round, for reporting.
6998pub fn worst_open(state: &RunState) -> Option<Severity> {
6999    state
7000        .reviews
7001        .last()?
7002        .reviews
7003        .iter()
7004        .flat_map(|r| r.findings.iter())
7005        .map(|f| f.severity)
7006        .max()
7007}
7008
7009#[cfg(test)]
7010mod tests {
7011    use super::*;
7012    use crate::run::GateStatus;
7013    use std::collections::BTreeMap;
7014    use std::time::Duration;
7015
7016    fn conductor() -> AgentSpec {
7017        AgentSpec {
7018            id: "conductor".to_owned(),
7019            kind: crate::config::AgentKind::Command,
7020            model: None,
7021            command: vec!["true".to_owned()],
7022            extra_args: Vec::new(),
7023            env: BTreeMap::new(),
7024            prompt_delivery: None,
7025        }
7026    }
7027
7028    fn spec(id: &str) -> AgentSpec {
7029        AgentSpec {
7030            id: id.to_owned(),
7031            kind: crate::config::AgentKind::Command,
7032            model: None,
7033            command: vec!["true".to_owned()],
7034            extra_args: Vec::new(),
7035            env: BTreeMap::new(),
7036            prompt_delivery: None,
7037        }
7038    }
7039
7040    // `next_untried_implementer` is the property `resume_quota_losses`'s own
7041    // fallback loop depends on to terminate: it must walk forward from the
7042    // seat's own position, never restart at the front of the roster, and it
7043    // must never hand back an id already tried, however many times that id
7044    // happens to appear.
7045
7046    #[test]
7047    fn next_untried_implementer_walks_forward_from_the_seats_own_position() {
7048        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
7049        let tried = BTreeSet::from(["beta".to_owned()]);
7050        // beta sits at index 1; the next candidate is gamma, never alpha —
7051        // which is very likely a different candidate slot's own agent.
7052        let next = next_untried_implementer(&roster, 1, &tried);
7053        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
7054    }
7055
7056    #[test]
7057    fn next_untried_implementer_does_not_wrap_back_past_its_own_start() {
7058        let roster = vec![spec("alpha"), spec("beta")];
7059        let tried = BTreeSet::from(["beta".to_owned()]);
7060        // beta is the roster's last entry: nothing follows it, and alpha —
7061        // earlier in the roster, almost certainly a different candidate
7062        // slot's own agent — must not be reached by wrapping back to it.
7063        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
7064    }
7065
7066    #[test]
7067    fn next_untried_implementer_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
7068        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
7069        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
7070        // beta (index 1) and gamma (index 2, the only entry after it) have
7071        // both been tried; alpha (index 0) never has, but it comes before
7072        // beta's own position, so there is nothing further for this seat.
7073        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
7074    }
7075
7076    #[test]
7077    fn next_untried_implementer_skips_ids_already_tried_even_when_duplicated() {
7078        let roster = vec![spec("a"), spec("a"), spec("b")];
7079        let tried = BTreeSet::from(["a".to_owned()]);
7080        let next = next_untried_implementer(&roster, 0, &tried);
7081        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
7082    }
7083
7084    #[test]
7085    fn next_untried_implementer_returns_none_once_every_id_is_tried() {
7086        let roster = vec![spec("a"), spec("b")];
7087        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
7088        assert!(next_untried_implementer(&roster, 0, &tried).is_none());
7089    }
7090
7091    #[test]
7092    fn remove_if_empty_only_ever_takes_a_bare_directory() {
7093        let dir = tempfile::tempdir().unwrap();
7094        let bay = dir.path().join("ffff");
7095
7096        // Not there yet: nothing to do, nothing to panic on.
7097        remove_if_empty(&bay);
7098        assert!(!bay.exists());
7099
7100        // Something still inside - the winner's worktree, or a stray file -
7101        // keeps the directory standing.
7102        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
7103        remove_if_empty(&bay);
7104        assert!(bay.exists(), "non-empty directory must survive");
7105
7106        // Once the last entry is gone, so is the directory itself.
7107        std::fs::remove_dir(bay.join("cand-A")).unwrap();
7108        remove_if_empty(&bay);
7109        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
7110    }
7111
7112    // `round_is_clean` is the exact decision this task fixed: a round with a
7113    // seat that never answered must not read the same as a round every seat
7114    // actually reviewed. These are deterministic and process-free by design —
7115    // the equivalent end-to-end check (a real reviewer timing out under a
7116    // live graph run) is a genuine race against wall-clock contention, and a
7117    // spawn slow enough to blow even a generous budget under a loaded test
7118    // run must not turn this specific regression check flaky.
7119
7120    #[test]
7121    fn a_full_panel_that_found_nothing_is_clean() {
7122        assert!(round_is_clean(
7123            0,
7124            true,
7125            2,
7126            2,
7127            0,
7128            IncompleteReviewPolicy::Block
7129        ));
7130    }
7131
7132    #[test]
7133    fn a_missing_seat_is_never_clean_under_the_default_policy() {
7134        assert!(!round_is_clean(
7135            0,
7136            true,
7137            1,
7138            2,
7139            0,
7140            IncompleteReviewPolicy::Block
7141        ));
7142    }
7143
7144    #[test]
7145    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
7146        assert!(!round_is_clean(
7147            1,
7148            true,
7149            1,
7150            2,
7151            0,
7152            IncompleteReviewPolicy::Warn
7153        ));
7154    }
7155
7156    #[test]
7157    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
7158        assert!(round_is_clean(
7159            0,
7160            true,
7161            1,
7162            2,
7163            0,
7164            IncompleteReviewPolicy::Warn
7165        ));
7166    }
7167
7168    #[test]
7169    fn a_full_panel_with_an_open_finding_is_not_clean() {
7170        assert!(!round_is_clean(
7171            1,
7172            true,
7173            2,
7174            2,
7175            0,
7176            IncompleteReviewPolicy::Block
7177        ));
7178    }
7179
7180    #[test]
7181    fn a_full_panel_with_a_red_e2e_is_not_clean() {
7182        assert!(!round_is_clean(
7183            0,
7184            false,
7185            2,
7186            2,
7187            0,
7188            IncompleteReviewPolicy::Block
7189        ));
7190    }
7191
7192    // The stall this task closes: under the default `block` policy, a seat
7193    // missing only because it was rate limited must not force a wait for a
7194    // session limit that will not lift by the next round. `round_is_clean`
7195    // is where that quorum carve-out lives; the review loop around it never
7196    // changes what a reviewer's vote or a finding's severity means.
7197
7198    #[test]
7199    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
7200        // 1 of 2 answered, and the one missing was quota'd — the exact
7201        // "review-2 rate limited (quota)" shape from the field report.
7202        assert!(round_is_clean(
7203            0,
7204            true,
7205            1,
7206            2,
7207            1,
7208            IncompleteReviewPolicy::Block
7209        ));
7210    }
7211
7212    #[test]
7213    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
7214        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
7215        // not a quota loss (`quota_missing` stays 0) — worth another try.
7216        assert!(!round_is_clean(
7217            0,
7218            true,
7219            1,
7220            2,
7221            0,
7222            IncompleteReviewPolicy::Block
7223        ));
7224    }
7225
7226    #[test]
7227    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
7228        assert!(!round_is_clean(
7229            1,
7230            true,
7231            1,
7232            2,
7233            1,
7234            IncompleteReviewPolicy::Block
7235        ));
7236        assert!(!round_is_clean(
7237            0,
7238            false,
7239            1,
7240            2,
7241            1,
7242            IncompleteReviewPolicy::Block
7243        ));
7244    }
7245
7246    #[test]
7247    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
7248        // Every seat quota'd, nobody answered: there is no panel to decide
7249        // on, so this must fall through to the existing block-and-retry
7250        // fallback rather than call an unreviewed patch clean.
7251        assert!(!round_is_clean(
7252            0,
7253            true,
7254            0,
7255            2,
7256            2,
7257            IncompleteReviewPolicy::Block
7258        ));
7259    }
7260
7261    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
7262        CommandOutcome {
7263            command: "test".to_owned(),
7264            code,
7265            output_tail: String::new(),
7266            duration_ms: 0,
7267            resource_blocked,
7268        }
7269    }
7270
7271    #[test]
7272    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
7273        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
7274        assert!(
7275            !verify_inconclusive(&[outcome(Some(1), false)]),
7276            "an ordinary failure is still evidence about the patch"
7277        );
7278        assert!(verify_inconclusive(&[outcome(None, true)]));
7279        assert!(
7280            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
7281            "one inconclusive outcome taints the whole batch"
7282        );
7283        assert!(!verify_inconclusive(&[]));
7284    }
7285
7286    #[tokio::test]
7287    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
7288        // Alive for the first two checks, then dead - confirms the loop
7289        // actually re-polls rather than deciding once and sleeping out the
7290        // ceiling regardless.
7291        let calls = std::sync::atomic::AtomicUsize::new(0);
7292        let started = Instant::now();
7293        wait_for_pids_with(
7294            &[123],
7295            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
7296            Duration::from_millis(5),
7297            Duration::from_secs(5),
7298        )
7299        .await;
7300        assert!(
7301            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
7302            "must keep checking rather than deciding on the first answer"
7303        );
7304        assert!(
7305            started.elapsed() < Duration::from_secs(1),
7306            "must return the moment it is confirmed dead, not wait out the ceiling"
7307        );
7308    }
7309
7310    #[tokio::test]
7311    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
7312        let started = Instant::now();
7313        wait_for_pids_with(
7314            &[123],
7315            |_| true, // never reports dead
7316            Duration::from_millis(5),
7317            Duration::from_millis(30),
7318        )
7319        .await;
7320        let elapsed = started.elapsed();
7321        assert!(
7322            elapsed >= Duration::from_millis(30),
7323            "must not give up before its own ceiling: {elapsed:?}"
7324        );
7325        assert!(
7326            elapsed < Duration::from_secs(1),
7327            "must not wait past its own ceiling either: {elapsed:?}"
7328        );
7329    }
7330
7331    #[tokio::test]
7332    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
7333        let started = Instant::now();
7334        wait_for_pids_with(
7335            &[],
7336            |_| true,
7337            Duration::from_secs(5),
7338            Duration::from_secs(5),
7339        )
7340        .await;
7341        assert!(
7342            started.elapsed() < Duration::from_millis(200),
7343            "an empty pid list has nothing to confirm"
7344        );
7345    }
7346
7347    // `review_conclusion` is the exact decision the review hand-off task
7348    // fixed: a round budget spent (or a tree that stopped moving) must not
7349    // collapse into `Blocked` regardless of what verification actually
7350    // said. Deterministic and process-free for the same reason the
7351    // `round_is_clean` family above is.
7352    fn review_round(
7353        clean: bool,
7354        blocking: usize,
7355        answered: usize,
7356        expected: usize,
7357        progressed: bool,
7358        e2e_ok: bool,
7359    ) -> ReviewRound {
7360        ReviewRound {
7361            round: 1,
7362            head: "h".to_owned(),
7363            verified_head: None,
7364            verified_at: None,
7365            reviews: Vec::new(),
7366            e2e: vec![CommandOutcome {
7367                command: "test".to_owned(),
7368                code: Some(if e2e_ok { 0 } else { 1 }),
7369                output_tail: String::new(),
7370                duration_ms: 0,
7371                resource_blocked: false,
7372            }],
7373            verify_retried: false,
7374            e2e_deferred: false,
7375            e2e_defer_reason: None,
7376            fix: None,
7377            blocking,
7378            answered,
7379            expected,
7380            clean,
7381            progressed,
7382            vote_split: false,
7383            reconsideration: Vec::new(),
7384            verdict: None,
7385        }
7386    }
7387
7388    #[test]
7389    fn review_conclusion_is_none_when_nothing_has_run() {
7390        assert_eq!(review_conclusion(&[], 3), None);
7391    }
7392
7393    #[test]
7394    fn review_conclusion_is_none_while_rounds_remain() {
7395        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
7396        assert_eq!(review_conclusion(&rounds, 3), None);
7397    }
7398
7399    #[test]
7400    fn review_conclusion_is_gating_once_a_round_is_clean() {
7401        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
7402        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
7403    }
7404
7405    #[test]
7406    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
7407        let rounds = vec![
7408            review_round(false, 1, 2, 2, true, true),
7409            review_round(false, 1, 2, 2, true, true),
7410        ];
7411        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
7412    }
7413
7414    #[test]
7415    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
7416        let rounds = vec![
7417            review_round(false, 1, 2, 2, true, true),
7418            review_round(false, 1, 2, 2, true, false),
7419        ];
7420        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
7421    }
7422
7423    #[test]
7424    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
7425        // Magi never got a command to run against this round's own head — a
7426        // resource-blocked attempt, not a red one — so this must never
7427        // settle on `Blocked` the way a genuine e2e failure would. `None`
7428        // here is what tells `Runner::review_loop` to retry the check
7429        // itself rather than trust this cheap recomputation with a verdict
7430        // it cannot actually produce.
7431        let mut blocked = review_round(false, 1, 2, 2, true, false);
7432        blocked.e2e[0].resource_blocked = true;
7433        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
7434        assert_eq!(review_conclusion(&rounds, 2), None);
7435    }
7436
7437    #[test]
7438    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
7439        // Missing input, not a verified tree — never a hand-off candidate.
7440        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
7441        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
7442    }
7443
7444    #[test]
7445    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
7446        let rounds = vec![
7447            review_round(false, 1, 2, 2, false, true),
7448            review_round(false, 1, 2, 2, false, true),
7449        ];
7450        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
7451    }
7452
7453    fn secs(n: u64) -> Duration {
7454        Duration::from_secs(n)
7455    }
7456
7457    /// A throwaway repo with one commit on `main`, for tests that need `merge`
7458    /// to make real (and, if it runs at all, real*ly fail*) git calls.
7459    fn init_repo(dir: &Path) {
7460        let run = |args: &[&str]| {
7461            let out = std::process::Command::new("git")
7462                .args(args)
7463                .current_dir(dir)
7464                .quiet()
7465                .output()
7466                .expect("spawn git");
7467            assert!(
7468                out.status.success(),
7469                "git {args:?} failed: {}",
7470                String::from_utf8_lossy(&out.stderr)
7471            );
7472        };
7473        run(&["init", "-b", "main"]);
7474        run(&["config", "user.name", "magi test"]);
7475        run(&["config", "user.email", "magi@example.com"]);
7476        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
7477        run(&["add", "-A"]);
7478        run(&["commit", "-m", "init"]);
7479    }
7480
7481    // `settle_questions` is what closes the ghost the phone showed: a run's
7482    // seat asked something, the run then ended, and nothing was left to
7483    // abandon the question it left `open`. `HOME` is a process-wide
7484    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
7485    // first time it runs in the binary — every test below still reaches the
7486    // same directory whichever call won, and each gets its own run id from
7487    // `RunState::new`, so they never collide there.
7488    fn ask_test_home() {
7489        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
7490    }
7491
7492    /// A minimal, git-free `Runner` at a given status — `settle_questions`
7493    /// reads nothing else off it.
7494    fn runner_at(status: RunStatus) -> Runner {
7495        let mut state = RunState::new(
7496            PathBuf::from("/nonexistent/repo"),
7497            "main".to_owned(),
7498            "deadbeef".to_owned(),
7499            "task".to_owned(),
7500            Config::default(),
7501        );
7502        state.status = status;
7503        Runner {
7504            state,
7505            roles: ResolvedRoles {
7506                implementers: Vec::new(),
7507                judges: Vec::new(),
7508                reviewers: Vec::new(),
7509                fixer: None,
7510                conductor: conductor(),
7511                implementer_roster: Vec::new(),
7512            },
7513            sem: Arc::new(Semaphore::new(1)),
7514            pause: Pause::new(),
7515            interrupt: Pause::new(),
7516        }
7517    }
7518
7519    /// `park_here` folding in the reason `Pause::park_because` recorded -
7520    /// this is what lets an operator reading a run's events tell an
7521    /// interrupt-driven park from an ordinary shutdown park.
7522    #[test]
7523    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
7524        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7525        let mut runner = runner_at(RunStatus::Implementing);
7526        let interrupt = Pause::new();
7527        runner.watch_interrupt(interrupt.clone());
7528
7529        interrupt.park_because("task a1b2 asked to run first");
7530
7531        assert!(runner.park_here().expect("park_here"));
7532        assert!(runner.state.parked);
7533        let last = runner.state.events.last().expect("a park event");
7534        assert_eq!(last.node, "park");
7535        assert!(
7536            last.message.contains("task a1b2 asked to run first"),
7537            "expected the interrupt reason in {:?}",
7538            last.message
7539        );
7540    }
7541
7542    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
7543    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
7544    /// cleared) must not make a *different* run - one only watching its own,
7545    /// unshared interrupt `Pause` - see itself as parked. If a future change
7546    /// ever collapsed these back into one handle, the interrupt scheduler
7547    /// would park every run for the rest of the daemon's life, not just the
7548    /// one it meant to interrupt.
7549    #[test]
7550    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
7551        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7552        let mut runner = runner_at(RunStatus::Implementing);
7553        let shutdown = Pause::new();
7554        runner.on_pause(shutdown.clone());
7555        let interrupt = Pause::new();
7556        runner.watch_interrupt(interrupt.clone());
7557
7558        // Nobody has asked for anything yet.
7559        assert!(!runner.park_here().expect("park_here"));
7560        assert!(!runner.state.parked);
7561
7562        // Only the interrupt handle fires; the shutdown handle stays clear.
7563        interrupt.park_because("test");
7564        assert!(!shutdown.parked());
7565        assert!(runner.park_here().expect("park_here"));
7566    }
7567
7568    /// The property every prior attempt at this feature failed to pin down:
7569    /// asking a run to park while one of its nodes has a real, in-flight
7570    /// async operation running (an agent call, in production) must not cut
7571    /// that operation short. `park_here` is only ever consulted *between*
7572    /// `execute`'s node calls - see its own doc - so nothing inside a node
7573    /// can observe a park request until the node itself returns. This proves
7574    /// that structurally, with real `tokio` concurrency and a channel
7575    /// handshake (never a sleep, which would only prove "usually", not
7576    /// "cannot"): the "node" below reports that it has genuinely started,
7577    /// and only then is the park requested; the node still has to be told to
7578    /// finish before `park_here` is ever called, exactly mirroring every
7579    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
7580    /// in `execute`.
7581    #[tokio::test]
7582    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
7583        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7584        let mut runner = runner_at(RunStatus::Implementing);
7585        let interrupt = Pause::new();
7586        runner.watch_interrupt(interrupt.clone());
7587
7588        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
7589        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
7590
7591        // Stands in for one node's in-flight agent call: it proves it has
7592        // genuinely started, then blocks - exactly as a spawned CLI process
7593        // does - until told to finish.
7594        let node = async move {
7595            started_tx.send(()).expect("send started");
7596            finish_rx.await.expect("recv finish");
7597            "node finished"
7598        };
7599
7600        let interrupter = async move {
7601            started_rx.await.expect("recv started");
7602            // The call is now genuinely in flight. Ask it to park.
7603            interrupt.park_because("higher-priority task waiting");
7604            // Nothing the node does can observe this yet - there is no
7605            // check inside it, by construction - so let the executor run
7606            // anything pending and then let the node finish on its own.
7607            tokio::task::yield_now().await;
7608            finish_tx.send(()).expect("send finish");
7609        };
7610
7611        let (node_result, ()) = tokio::join!(node, interrupter);
7612        assert_eq!(
7613            node_result, "node finished",
7614            "the in-flight call ran to completion"
7615        );
7616
7617        // Only now, at the boundary the real `execute` would check right
7618        // after this node, does the park take effect.
7619        assert!(runner.park_here().expect("park_here"));
7620        assert!(runner.state.parked);
7621    }
7622
7623    /// A run parked mid-competition carries every field it had accumulated
7624    /// through the exact same disk round-trip an ordinary resume uses -
7625    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
7626    /// Nothing about parking for an interrupt is a special case of that path;
7627    /// this is what proves it rather than assuming it.
7628    #[test]
7629    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
7630        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7631        let mut runner = runner_at(RunStatus::Judging);
7632        // `Runner::resume` re-resolves roles from the saved config, which
7633        // refuses an empty roster - give it the same minimal one `conductor`
7634        // itself uses.
7635        runner.state.config.agents = vec![conductor()];
7636        runner.state.candidates = vec![Candidate {
7637            index: 0,
7638            label: 'A',
7639            agent: "alpha".to_owned(),
7640            branch: "magi/x/A".to_owned(),
7641            worktree: PathBuf::from("/nonexistent/worktree"),
7642            summary: "did the thing".to_owned(),
7643            stat: "1 file changed".to_owned(),
7644            files: 1,
7645            commits: 1,
7646            empty: false,
7647            failed: None,
7648            verified_noop: None,
7649            duration_ms: 1234,
7650            folded: false,
7651        }];
7652        let run_id = runner.state.id.clone();
7653
7654        let interrupt = Pause::new();
7655        runner.watch_interrupt(interrupt.clone());
7656        interrupt.park_because("task c3d4 asked to run first");
7657        assert!(runner.park_here().expect("park_here"));
7658
7659        let resumed = Runner::resume(&run_id).expect("resume");
7660        assert_eq!(resumed.state.candidates.len(), 1);
7661        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
7662        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
7663        assert_eq!(resumed.state.status, runner.state.status);
7664        assert!(
7665            resumed.state.parked,
7666            "still parked until `execute` actually walks the graph again"
7667        );
7668        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
7669    }
7670
7671    /// A fresh open question on `run`, stored and handed back for assertions.
7672    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
7673        let mut q = ask::Question::new(
7674            run.to_owned(),
7675            "implement".to_owned(),
7676            "impl-A".to_owned(),
7677            "Which storage backend should the cache use?".to_owned(),
7678            String::new(),
7679            vec!["SQLite".to_owned(), "Redis".to_owned()],
7680        );
7681        store.put(&mut q).unwrap();
7682        q
7683    }
7684
7685    #[test]
7686    fn a_failed_runs_open_question_is_abandoned() {
7687        ask_test_home();
7688        let store = ask::Questions::open();
7689        let mut runner = runner_at(RunStatus::Failed);
7690        let run = runner.state.id.clone();
7691        let q = ask_open_question(&store, &run);
7692
7693        runner.settle_questions();
7694
7695        let back = store.get(&q.id).unwrap();
7696        assert!(
7697            !back.status.open(),
7698            "the seat that asked died with the run; nobody is left to read an answer"
7699        );
7700        assert!(
7701            back.detail.contains(&run) && back.detail.contains("failed"),
7702            "the reason names what the run became, not just that it is gone: {}",
7703            back.detail
7704        );
7705    }
7706
7707    #[test]
7708    fn a_merged_runs_open_question_is_abandoned_too() {
7709        ask_test_home();
7710        let store = ask::Questions::open();
7711        // A run that finishes cleanly still leaves nobody to read an answer -
7712        // this is not only a failure-path cleanup.
7713        for status in [RunStatus::Merged, RunStatus::Ready] {
7714            let mut runner = runner_at(status);
7715            let run = runner.state.id.clone();
7716            let q = ask_open_question(&store, &run);
7717
7718            runner.settle_questions();
7719
7720            let back = store.get(&q.id).unwrap();
7721            assert!(
7722                !back.status.open(),
7723                "{status:?} run's question must not outlive the run"
7724            );
7725        }
7726    }
7727
7728    #[test]
7729    fn a_still_resumable_runs_open_question_is_left_alone() {
7730        ask_test_home();
7731        let store = ask::Questions::open();
7732        // `Blocked` and `Stalled` can still be resumed — the candidates, the
7733        // review round and the seat sessions are all still on disk — so a
7734        // question asked mid-round may yet get a real answer from a real
7735        // resume. Sweeping it here would be exactly the failure mode this
7736        // whole feature exists to avoid on the other side.
7737        for status in [RunStatus::Blocked, RunStatus::Stalled] {
7738            let mut runner = runner_at(status);
7739            let run = runner.state.id.clone();
7740            let q = ask_open_question(&store, &run);
7741
7742            runner.settle_questions();
7743
7744            let back = store.get(&q.id).unwrap();
7745            assert!(
7746                back.status.open(),
7747                "{status:?} is still alive; the question must still be waiting"
7748            );
7749        }
7750    }
7751
7752    #[test]
7753    fn settle_questions_never_touches_an_already_answered_question() {
7754        ask_test_home();
7755        let store = ask::Questions::open();
7756        let mut runner = runner_at(RunStatus::Failed);
7757        let run = runner.state.id.clone();
7758        let mut q = ask_open_question(&store, &run);
7759        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
7760            .unwrap();
7761        store.put(&mut q).unwrap();
7762
7763        // Called twice, the way a crash-recovered daemon reclaim and the
7764        // graph's own cleanup both can for the same run — `abandon_for_run`
7765        // only ever touches what is still open, so this must be inert both
7766        // times, not merely the second.
7767        runner.settle_questions();
7768        runner.settle_questions();
7769
7770        let back = store.get(&q.id).unwrap();
7771        assert_eq!(
7772            back.status,
7773            ask::QuestionStatus::Answered,
7774            "a real answer is a decision on record, never overwritten by a sweep"
7775        );
7776    }
7777
7778    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
7779    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
7780    /// without merging, whose winner is still the operator's answer to read.
7781    /// Nothing previously called `fold_run` itself with a real `tally`, so
7782    /// this is the first test to pin down the one distinction the whole
7783    /// automatic-fold feature depends on: the winner's worktree and branch
7784    /// must survive, everything else sharing the run's worktree bay - a
7785    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
7786    /// second sweep treats every non-winner directory under the bay alike -
7787    /// must not.
7788    #[tokio::test]
7789    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
7790        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
7791        let tmp = tempfile::tempdir().expect("tempdir");
7792        let repo = tmp.path().join("repo");
7793        std::fs::create_dir_all(&repo).unwrap();
7794        init_repo(&repo);
7795
7796        let mut config = Config::default();
7797        config.graph.worktree_root = Some(tmp.path().join("wt"));
7798
7799        let mut state = RunState::new(
7800            repo.clone(),
7801            "main".to_owned(),
7802            "deadbeef".to_owned(),
7803            "task".to_owned(),
7804            config,
7805        );
7806        let root = state.worktree_root();
7807        let wt_a = root.join("cand-A");
7808        let wt_b = root.join("cand-B");
7809        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
7810            .await
7811            .expect("worktree A");
7812        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
7813            .await
7814            .expect("worktree B");
7815
7816        state.candidates = vec![
7817            Candidate {
7818                index: 0,
7819                label: 'A',
7820                agent: "alpha".to_owned(),
7821                branch: "magi/x/A".to_owned(),
7822                worktree: wt_a.clone(),
7823                summary: String::new(),
7824                stat: String::new(),
7825                files: 0,
7826                commits: 0,
7827                empty: false,
7828                failed: None,
7829                verified_noop: None,
7830                duration_ms: 0,
7831                folded: false,
7832            },
7833            Candidate {
7834                index: 1,
7835                label: 'B',
7836                agent: "beta".to_owned(),
7837                branch: "magi/x/B".to_owned(),
7838                worktree: wt_b.clone(),
7839                summary: String::new(),
7840                stat: String::new(),
7841                files: 0,
7842                commits: 0,
7843                empty: false,
7844                failed: None,
7845                verified_noop: None,
7846                duration_ms: 0,
7847                folded: false,
7848            },
7849        ];
7850        state.tally = Some(Tally {
7851            first_choice: BTreeMap::from([('A', 1)]),
7852            borda: BTreeMap::new(),
7853            winner: 'A',
7854            rankings: 1,
7855            unanimous_initial: true,
7856            deliberated: false,
7857            changed_votes: 0,
7858            unanimous_final: true,
7859            tie_break: None,
7860            judges: 1,
7861            present: 1,
7862            quorum: 1,
7863            met_quorum: true,
7864            uncontested: None,
7865        });
7866        state.status = RunStatus::Ready;
7867
7868        fold_run(&mut state, false, &crate::run::home())
7869            .await
7870            .expect("fold_run");
7871
7872        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
7873        assert!(
7874            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7875            "the unmerged winner's branch survives"
7876        );
7877        assert!(
7878            !state.candidates[0].folded,
7879            "the winner is not marked folded"
7880        );
7881
7882        assert!(!wt_b.exists(), "the loser's worktree is removed");
7883        assert!(
7884            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
7885            "the loser's branch is removed"
7886        );
7887        assert!(state.candidates[1].folded, "the loser is marked folded");
7888    }
7889
7890    /// A branch handed to a later run is that run's (and its pull request's):
7891    /// folding the run that released it must not delete it.
7892    #[tokio::test]
7893    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
7894        let tmp = tempfile::tempdir().expect("tempdir");
7895        let repo = tmp.path().join("repo");
7896        std::fs::create_dir_all(&repo).unwrap();
7897        init_repo(&repo);
7898        let home = tmp.path().join("home");
7899
7900        let mut config = Config::default();
7901        config.graph.worktree_root = Some(tmp.path().join("wt"));
7902        let mut state = RunState::new(
7903            repo.clone(),
7904            "main".to_owned(),
7905            "deadbeef".to_owned(),
7906            "task".to_owned(),
7907            config,
7908        );
7909        // The worktree is already gone (released); the branch survives.
7910        git::git(&repo, &["branch", "magi/x/A", "main"])
7911            .await
7912            .expect("branch");
7913        state.candidates = vec![Candidate {
7914            index: 0,
7915            label: 'A',
7916            agent: "alpha".to_owned(),
7917            branch: "magi/x/A".to_owned(),
7918            worktree: state.worktree_root().join("cand-A"),
7919            summary: String::new(),
7920            stat: String::new(),
7921            files: 0,
7922            commits: 0,
7923            empty: false,
7924            failed: None,
7925            verified_noop: None,
7926            duration_ms: 0,
7927            folded: true,
7928        }];
7929        state.released_to = Some("20260901-000000-new1".to_owned());
7930        state.released_branches = vec!["magi/x/A".to_owned()];
7931
7932        fold_run(&mut state, true, &home).await.expect("fold_run");
7933
7934        assert!(
7935            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7936            "the handed-over branch survives a fold"
7937        );
7938    }
7939
7940    /// `status == Ready` used to be read as "this is the harmless
7941    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
7942    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
7943    /// run's PR was closed without merging — and reentering `merge` with
7944    /// `mode` still `Pr` does not know the difference, so it pushed and
7945    /// opened a second pull request. `mode == Local` reproduces the same
7946    /// blind spot without a network call: reentry must not attempt another
7947    /// git merge once this node has already recorded an outcome.
7948    #[tokio::test]
7949    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
7950        let tmp = tempfile::tempdir().expect("tempdir");
7951        let repo = tmp.path().join("repo");
7952        std::fs::create_dir_all(&repo).unwrap();
7953        init_repo(&repo);
7954
7955        let mut config = Config::default();
7956        config.merge.mode = MergeMode::Local;
7957
7958        let mut state = RunState::new(
7959            repo.clone(),
7960            "main".to_owned(),
7961            "deadbeef".to_owned(),
7962            "task".to_owned(),
7963            config,
7964        );
7965        state.candidates = vec![Candidate {
7966            index: 0,
7967            label: 'A',
7968            agent: "alpha".to_owned(),
7969            branch: "does-not-exist".to_owned(),
7970            worktree: repo.clone(),
7971            summary: String::new(),
7972            stat: String::new(),
7973            files: 0,
7974            commits: 0,
7975            empty: false,
7976            failed: None,
7977            verified_noop: None,
7978            duration_ms: 0,
7979            folded: false,
7980        }];
7981        state.tally = Some(Tally {
7982            first_choice: BTreeMap::from([('A', 1)]),
7983            borda: BTreeMap::new(),
7984            winner: 'A',
7985            rankings: 1,
7986            unanimous_initial: true,
7987            deliberated: false,
7988            changed_votes: 0,
7989            unanimous_final: true,
7990            tie_break: None,
7991            judges: 0,
7992            present: 0,
7993            quorum: 0,
7994            met_quorum: true,
7995            uncontested: Some("only candidate A produced a change".to_owned()),
7996        });
7997        state.reviews = vec![ReviewRound {
7998            round: 1,
7999            head: "deadbeef".to_owned(),
8000            verified_head: None,
8001            verified_at: None,
8002            reviews: Vec::new(),
8003            e2e: Vec::new(),
8004            fix: None,
8005            blocking: 0,
8006            answered: 0,
8007            expected: 0,
8008            clean: true,
8009            verify_retried: false,
8010            e2e_deferred: false,
8011            e2e_defer_reason: None,
8012            progressed: false,
8013            vote_split: false,
8014            reconsideration: Vec::new(),
8015            verdict: None,
8016        }];
8017        state.gate = vec![CommandOutcome {
8018            command: "test".to_owned(),
8019            code: Some(0),
8020            output_tail: String::new(),
8021            duration_ms: 0,
8022            resource_blocked: false,
8023        }];
8024        state.gate_ran = true;
8025        // Reached its conclusion already — e.g. `land` closing the PR without
8026        // merging it, which (like the honest `MergeMode::None` path) leaves
8027        // `status` at `Ready`. The recorded outcome is what actually marks
8028        // this node done.
8029        state.status = RunStatus::Ready;
8030        state.merge = Some(MergeOutcome {
8031            mode: MergeMode::Local,
8032            ok: false,
8033            detail: "already concluded".to_owned(),
8034        });
8035
8036        let mut runner = Runner {
8037            state,
8038            roles: ResolvedRoles {
8039                implementers: Vec::new(),
8040                judges: Vec::new(),
8041                reviewers: Vec::new(),
8042                fixer: None,
8043                conductor: conductor(),
8044                implementer_roster: Vec::new(),
8045            },
8046            sem: Arc::new(Semaphore::new(1)),
8047            pause: Pause::new(),
8048            interrupt: Pause::new(),
8049        };
8050
8051        runner.merge().await.expect("merge");
8052
8053        assert_eq!(
8054            runner.state.status,
8055            RunStatus::Ready,
8056            "a concluded run's status must not change on reentry"
8057        );
8058        assert_eq!(
8059            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
8060            Some("already concluded"),
8061            "merge must not run again once the node already recorded an outcome"
8062        );
8063    }
8064
8065    /// `gate` leaves `state.gate_ran` false both before it has ever run and
8066    /// when its last attempt was resource-blocked (the shared build cache
8067    /// could not be acquired or confirmed fresh in time - see
8068    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
8069    /// `Vec` this also leaves behind used to read as "nothing failed" and let
8070    /// a run merge a tree the gate never actually checked - exactly the case
8071    /// a contended cache produces on every retry until it clears. `merge`
8072    /// must refuse until `gate` has actually recorded an attempt.
8073    #[tokio::test]
8074    async fn merge_refuses_a_gate_that_has_not_actually_run() {
8075        let tmp = tempfile::tempdir().expect("tempdir");
8076        let repo = tmp.path().join("repo");
8077        std::fs::create_dir_all(&repo).unwrap();
8078        init_repo(&repo);
8079
8080        let mut config = Config::default();
8081        config.merge.mode = MergeMode::Local;
8082
8083        let mut state = RunState::new(
8084            repo.clone(),
8085            "main".to_owned(),
8086            "deadbeef".to_owned(),
8087            "task".to_owned(),
8088            config,
8089        );
8090        state.candidates = vec![Candidate {
8091            index: 0,
8092            label: 'A',
8093            agent: "alpha".to_owned(),
8094            branch: "does-not-exist".to_owned(),
8095            worktree: repo.clone(),
8096            summary: String::new(),
8097            stat: String::new(),
8098            files: 0,
8099            commits: 0,
8100            empty: false,
8101            failed: None,
8102            verified_noop: None,
8103            duration_ms: 0,
8104            folded: false,
8105        }];
8106        state.tally = Some(Tally {
8107            first_choice: BTreeMap::from([('A', 1)]),
8108            borda: BTreeMap::new(),
8109            winner: 'A',
8110            rankings: 1,
8111            unanimous_initial: true,
8112            deliberated: false,
8113            changed_votes: 0,
8114            unanimous_final: true,
8115            tie_break: None,
8116            judges: 0,
8117            present: 0,
8118            quorum: 0,
8119            met_quorum: true,
8120            uncontested: Some("only candidate A produced a change".to_owned()),
8121        });
8122        state.reviews = vec![ReviewRound {
8123            round: 1,
8124            head: "deadbeef".to_owned(),
8125            verified_head: None,
8126            verified_at: None,
8127            reviews: Vec::new(),
8128            e2e: Vec::new(),
8129            fix: None,
8130            blocking: 0,
8131            answered: 0,
8132            expected: 0,
8133            clean: true,
8134            verify_retried: false,
8135            e2e_deferred: false,
8136            e2e_defer_reason: None,
8137            progressed: false,
8138            vote_split: false,
8139            reconsideration: Vec::new(),
8140            verdict: None,
8141        }];
8142        // The point: `gate` has not recorded anything yet.
8143        state.gate = Vec::new();
8144        state.gate_ran = false;
8145        state.status = RunStatus::Gating;
8146
8147        let mut runner = Runner {
8148            state,
8149            roles: ResolvedRoles {
8150                implementers: Vec::new(),
8151                judges: Vec::new(),
8152                reviewers: Vec::new(),
8153                fixer: None,
8154                conductor: conductor(),
8155                implementer_roster: Vec::new(),
8156            },
8157            sem: Arc::new(Semaphore::new(1)),
8158            pause: Pause::new(),
8159            interrupt: Pause::new(),
8160        };
8161
8162        runner.merge().await.expect("merge");
8163
8164        assert!(
8165            runner.state.merge.is_none(),
8166            "an empty gate must never be read as a passing one: {:?}",
8167            runner.state.merge
8168        );
8169    }
8170
8171    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
8172    /// commands configured and `merge.mode` is `none` (a review-only run).
8173    /// `gate` must still record a real attempt — zero commands, vacuously
8174    /// passed — rather than leaving `state.gate` empty in a way `merge`
8175    /// cannot tell apart from "never ran"; otherwise the run reaches
8176    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
8177    #[tokio::test]
8178    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
8179        let tmp = tempfile::tempdir().expect("tempdir");
8180        let repo = tmp.path().join("repo");
8181        std::fs::create_dir_all(&repo).unwrap();
8182        init_repo(&repo);
8183
8184        // Default config: `verify.gate` empty, `merge.mode` is `none`.
8185        let config = Config::default();
8186
8187        let mut state = RunState::new(
8188            repo.clone(),
8189            "main".to_owned(),
8190            "deadbeef".to_owned(),
8191            "task".to_owned(),
8192            config,
8193        );
8194        state.candidates = vec![Candidate {
8195            index: 0,
8196            label: 'A',
8197            agent: "alpha".to_owned(),
8198            branch: "does-not-exist".to_owned(),
8199            worktree: repo.clone(),
8200            summary: String::new(),
8201            stat: String::new(),
8202            files: 0,
8203            commits: 0,
8204            empty: false,
8205            failed: None,
8206            verified_noop: None,
8207            duration_ms: 0,
8208            folded: false,
8209        }];
8210        state.tally = Some(Tally {
8211            first_choice: BTreeMap::from([('A', 1)]),
8212            borda: BTreeMap::new(),
8213            winner: 'A',
8214            rankings: 1,
8215            unanimous_initial: true,
8216            deliberated: false,
8217            changed_votes: 0,
8218            unanimous_final: true,
8219            tie_break: None,
8220            judges: 0,
8221            present: 0,
8222            quorum: 0,
8223            met_quorum: true,
8224            uncontested: Some("only candidate A produced a change".to_owned()),
8225        });
8226        state.reviews = vec![ReviewRound {
8227            round: 1,
8228            head: "deadbeef".to_owned(),
8229            verified_head: None,
8230            verified_at: None,
8231            reviews: Vec::new(),
8232            e2e: Vec::new(),
8233            fix: None,
8234            blocking: 0,
8235            answered: 0,
8236            expected: 0,
8237            clean: true,
8238            verify_retried: false,
8239            e2e_deferred: false,
8240            e2e_defer_reason: None,
8241            progressed: false,
8242            vote_split: false,
8243            reconsideration: Vec::new(),
8244            verdict: None,
8245        }];
8246
8247        let mut runner = Runner {
8248            state,
8249            roles: ResolvedRoles {
8250                implementers: Vec::new(),
8251                judges: Vec::new(),
8252                reviewers: Vec::new(),
8253                fixer: None,
8254                conductor: conductor(),
8255                implementer_roster: Vec::new(),
8256            },
8257            sem: Arc::new(Semaphore::new(1)),
8258            pause: Pause::new(),
8259            interrupt: Pause::new(),
8260        };
8261
8262        runner.gate().await.expect("gate");
8263        assert!(
8264            runner.state.gate_ran,
8265            "zero configured commands is still a real attempt, not an unrun gate"
8266        );
8267        assert!(runner.state.gate.is_empty());
8268        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
8269        assert_ne!(
8270            runner.state.status,
8271            RunStatus::Blocked,
8272            "a gate with nothing to check must not read as failed"
8273        );
8274
8275        runner.merge().await.expect("merge");
8276        assert_eq!(
8277            runner.state.status,
8278            RunStatus::Ready,
8279            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
8280        );
8281    }
8282
8283    /// `Config::cache_dir` is derived from `verify.e2e` as well as
8284    /// `verify.gate` (so the e2e leg and the final gate never build against
8285    /// different directories). With zero `verify.gate` commands but a
8286    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
8287    /// that lease before discovering it had nothing to run - so a repo with
8288    /// no gate commands could come back `resource_blocked` (and therefore
8289    /// still `gate_ran == false`) on nothing but an unrelated run holding the
8290    /// cache, exactly the contention this run's own zero commands could
8291    /// never have touched. `gate` must recognise there is nothing to check
8292    /// before it ever asks for the lease.
8293    #[tokio::test]
8294    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
8295        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8296        let home = crate::run::home();
8297
8298        let tmp = tempfile::tempdir().expect("tempdir");
8299        let repo = tmp.path().join("repo");
8300        std::fs::create_dir_all(&repo).unwrap();
8301        init_repo(&repo);
8302        // Unique to this test, so holding its lease cannot collide with
8303        // another test sharing the same process-wide `home`.
8304        let cache_dir = tmp.path().join("target");
8305
8306        let mut config = Config::default();
8307        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
8308        // `verify.gate` stays empty (the default). Bounded so a regression
8309        // that does start waiting fails the test in seconds, not hangs it.
8310        config.graph.timeout_verify = Some(2);
8311
8312        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8313        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8314            .expect("no io error acquiring directly")
8315        {
8316            crate::cache::AcquireOutcome::Acquired(g) => g,
8317            crate::cache::AcquireOutcome::Busy(b) => {
8318                panic!("expected the direct acquire to win the lease first: {b:?}")
8319            }
8320        };
8321
8322        let mut state = RunState::new(
8323            repo.clone(),
8324            "main".to_owned(),
8325            "deadbeef".to_owned(),
8326            "task".to_owned(),
8327            config,
8328        );
8329        state.candidates = vec![Candidate {
8330            index: 0,
8331            label: 'A',
8332            agent: "alpha".to_owned(),
8333            branch: "does-not-exist".to_owned(),
8334            worktree: repo.clone(),
8335            summary: String::new(),
8336            stat: String::new(),
8337            files: 0,
8338            commits: 0,
8339            empty: false,
8340            failed: None,
8341            verified_noop: None,
8342            duration_ms: 0,
8343            folded: false,
8344        }];
8345        state.tally = Some(Tally {
8346            first_choice: BTreeMap::from([('A', 1)]),
8347            borda: BTreeMap::new(),
8348            winner: 'A',
8349            rankings: 1,
8350            unanimous_initial: true,
8351            deliberated: false,
8352            changed_votes: 0,
8353            unanimous_final: true,
8354            tie_break: None,
8355            judges: 0,
8356            present: 0,
8357            quorum: 0,
8358            met_quorum: true,
8359            uncontested: Some("only candidate A produced a change".to_owned()),
8360        });
8361        state.reviews = vec![ReviewRound {
8362            round: 1,
8363            head: "deadbeef".to_owned(),
8364            verified_head: None,
8365            verified_at: None,
8366            reviews: Vec::new(),
8367            e2e: Vec::new(),
8368            fix: None,
8369            blocking: 0,
8370            answered: 0,
8371            expected: 0,
8372            clean: true,
8373            verify_retried: false,
8374            e2e_deferred: false,
8375            e2e_defer_reason: None,
8376            progressed: false,
8377            vote_split: false,
8378            reconsideration: Vec::new(),
8379            verdict: None,
8380        }];
8381
8382        let mut runner = Runner {
8383            state,
8384            roles: ResolvedRoles {
8385                implementers: Vec::new(),
8386                judges: Vec::new(),
8387                reviewers: Vec::new(),
8388                fixer: None,
8389                conductor: conductor(),
8390                implementer_roster: Vec::new(),
8391            },
8392            sem: Arc::new(Semaphore::new(1)),
8393            pause: Pause::new(),
8394            interrupt: Pause::new(),
8395        };
8396
8397        let started = std::time::Instant::now();
8398        runner.gate().await.expect("gate");
8399        assert!(
8400            started.elapsed() < Duration::from_secs(1),
8401            "a gate with nothing to run must never wait on a lease it never needed"
8402        );
8403        assert!(
8404            runner.state.gate_ran,
8405            "zero commands is still a real, immediate attempt"
8406        );
8407        assert!(runner.state.gate.is_empty());
8408        assert_ne!(
8409            runner.state.status,
8410            RunStatus::Blocked,
8411            "must not read as resource-blocked on a lease it never asked for"
8412        );
8413    }
8414
8415    /// The addendum's second gap: a `verify.gate` command running for real
8416    /// wall-clock time had nothing at all to show for it in `active` before
8417    /// `run_commands` learned to record it — a run could sit in `Gating` for
8418    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
8419    /// what was actually happening. Proven with a genuinely still-running
8420    /// command, not just a before/after check on the final state: a poller
8421    /// task reads the same `run.json` `gate()` is writing, the same way the
8422    /// phone or `magi show` would, while the shell command is still blocked
8423    /// on its own release marker.
8424    #[tokio::test]
8425    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
8426        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8427
8428        let tmp = tempfile::tempdir().expect("tempdir");
8429        let repo = tmp.path().join("repo");
8430        std::fs::create_dir_all(&repo).unwrap();
8431        init_repo(&repo);
8432
8433        let mut config = Config::default();
8434        config.verify.gate = vec![
8435            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
8436             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
8437                .to_owned(),
8438        ];
8439
8440        let mut state = RunState::new(
8441            repo.clone(),
8442            "main".to_owned(),
8443            "deadbeef".to_owned(),
8444            "task".to_owned(),
8445            config,
8446        );
8447        let run_id = state.id.clone();
8448        state.candidates = vec![Candidate {
8449            index: 0,
8450            label: 'A',
8451            agent: "alpha".to_owned(),
8452            branch: "does-not-exist".to_owned(),
8453            worktree: repo.clone(),
8454            summary: String::new(),
8455            stat: String::new(),
8456            files: 0,
8457            commits: 0,
8458            empty: false,
8459            failed: None,
8460            verified_noop: None,
8461            duration_ms: 0,
8462            folded: false,
8463        }];
8464        state.tally = Some(Tally {
8465            first_choice: BTreeMap::from([('A', 1)]),
8466            borda: BTreeMap::new(),
8467            winner: 'A',
8468            rankings: 1,
8469            unanimous_initial: true,
8470            deliberated: false,
8471            changed_votes: 0,
8472            unanimous_final: true,
8473            tie_break: None,
8474            judges: 0,
8475            present: 0,
8476            quorum: 0,
8477            met_quorum: true,
8478            uncontested: Some("only candidate A produced a change".to_owned()),
8479        });
8480        state.reviews = vec![ReviewRound {
8481            round: 1,
8482            head: "deadbeef".to_owned(),
8483            verified_head: None,
8484            verified_at: None,
8485            reviews: Vec::new(),
8486            e2e: Vec::new(),
8487            fix: None,
8488            blocking: 0,
8489            answered: 0,
8490            expected: 0,
8491            clean: true,
8492            verify_retried: false,
8493            e2e_deferred: false,
8494            e2e_defer_reason: None,
8495            progressed: false,
8496            vote_split: false,
8497            reconsideration: Vec::new(),
8498            verdict: None,
8499        }];
8500
8501        let mut runner = Runner {
8502            state,
8503            roles: ResolvedRoles {
8504                implementers: Vec::new(),
8505                judges: Vec::new(),
8506                reviewers: Vec::new(),
8507                fixer: None,
8508                conductor: conductor(),
8509                implementer_roster: Vec::new(),
8510            },
8511            sem: Arc::new(Semaphore::new(1)),
8512            pause: Pause::new(),
8513            interrupt: Pause::new(),
8514        };
8515
8516        let started_marker = repo.join("started.marker");
8517        let release_marker = repo.join("release.marker");
8518        let poller = tokio::spawn(async move {
8519            // Bounded so a regression that never records the task entry
8520            // fails this test in seconds instead of hanging the suite —
8521            // the same shape `a_park_requested_while_a_seat_is_mid_call_
8522            // does_not_cut_it_short` uses for the same reason.
8523            for _ in 0..100 {
8524                if started_marker.exists()
8525                    && let Ok(s) = crate::run::RunState::load(&run_id)
8526                    && let Some(a) = s.active.get("gate")
8527                {
8528                    std::fs::write(&release_marker, b"go").expect("release marker");
8529                    return Some(a.clone());
8530                }
8531                tokio::time::sleep(Duration::from_millis(50)).await;
8532            }
8533            None
8534        });
8535
8536        runner.gate().await.expect("gate");
8537        let captured = poller.await.expect("poller task");
8538        let captured = captured.expect(
8539            "the poller never saw a `gate` task entry in run.json while the command was \
8540             still blocked on its own release marker",
8541        );
8542
8543        assert_eq!(captured.task.as_deref(), Some("gate"));
8544        assert_eq!(captured.node, "gate");
8545        assert_eq!(captured.index, Some(1));
8546        assert_eq!(captured.total, Some(1));
8547        assert!(
8548            captured
8549                .command
8550                .as_deref()
8551                .is_some_and(|c| c.contains("started.marker")),
8552            "{captured:?}"
8553        );
8554
8555        assert!(
8556            runner.state.active.is_empty(),
8557            "the entry must be cleared once the command actually finished: {:?}",
8558            runner.state.active
8559        );
8560        assert!(runner.state.gate_ran);
8561        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
8562    }
8563
8564    /// The shape the incident this whole fix responds to actually had: the
8565    /// round budget spent, the last round's own e2e blocked on the shared
8566    /// build cache (held here by a live pid — this test process — exactly
8567    /// `cache`'s own unit tests' pattern for "another owner, still alive"
8568    /// without forking a process). `stop_reviewing` must retry it — not
8569    /// silently leave the round looking untouched (the catch-up-only half of
8570    /// the bug), and not read the contention as a red `e2e` and block the
8571    /// run on it (the other half). Called directly, the same way
8572    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
8573    /// above exercises `gate`, so this never needs a real cargo build to
8574    /// reach: the lease is never released, so `with_cache_lease` never gets
8575    /// past acquiring it into anything that would need a real workspace.
8576    #[tokio::test]
8577    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
8578        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8579        let home = crate::run::home();
8580
8581        let tmp = tempfile::tempdir().expect("tempdir");
8582        let repo = tmp.path().join("repo");
8583        std::fs::create_dir_all(&repo).unwrap();
8584        init_repo(&repo);
8585        let head = crate::git::rev_parse(&repo, "HEAD")
8586            .await
8587            .expect("rev-parse");
8588        // Unique to this test, so holding its lease cannot collide with
8589        // another test sharing the same process-wide `home`.
8590        let cache_dir = tmp.path().join("target");
8591
8592        let mut config = Config::default();
8593        config.verify.e2e = vec![format!(
8594            "CARGO_TARGET_DIR='{}' test -f README.md",
8595            cache_dir.display()
8596        )];
8597        config.graph.review_rounds = 1;
8598        // Bounded so a regression that does start waiting fails the test in
8599        // seconds, not hangs it.
8600        config.graph.timeout_verify = Some(2);
8601
8602        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8603        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8604            .expect("no io error acquiring directly")
8605        {
8606            crate::cache::AcquireOutcome::Acquired(g) => g,
8607            crate::cache::AcquireOutcome::Busy(b) => {
8608                panic!("expected the direct acquire to win the lease first: {b:?}")
8609            }
8610        };
8611
8612        let mut state = RunState::new(
8613            repo.clone(),
8614            "main".to_owned(),
8615            head.clone(),
8616            "task".to_owned(),
8617            config,
8618        );
8619        state.candidates = vec![Candidate {
8620            index: 0,
8621            label: 'A',
8622            agent: "alpha".to_owned(),
8623            branch: "does-not-exist".to_owned(),
8624            worktree: repo.clone(),
8625            summary: String::new(),
8626            stat: String::new(),
8627            files: 0,
8628            commits: 0,
8629            empty: false,
8630            failed: None,
8631            verified_noop: None,
8632            duration_ms: 0,
8633            folded: false,
8634        }];
8635        state.tally = Some(Tally {
8636            first_choice: BTreeMap::from([('A', 1)]),
8637            borda: BTreeMap::new(),
8638            winner: 'A',
8639            rankings: 1,
8640            unanimous_initial: true,
8641            deliberated: false,
8642            changed_votes: 0,
8643            unanimous_final: true,
8644            tie_break: None,
8645            judges: 0,
8646            present: 0,
8647            quorum: 0,
8648            met_quorum: true,
8649            uncontested: Some("only candidate A produced a change".to_owned()),
8650        });
8651        // The round budget's last round, deferred: `needs_catchup_run`'s
8652        // other trigger. `stop_reviewing`'s retry machinery must treat this
8653        // exactly like a resource-blocked attempt once it actually runs.
8654        state.reviews = vec![ReviewRound {
8655            round: 1,
8656            head: head.clone(),
8657            verified_head: None,
8658            verified_at: None,
8659            reviews: Vec::new(),
8660            e2e: Vec::new(),
8661            fix: None,
8662            blocking: 1,
8663            answered: 1,
8664            expected: 1,
8665            clean: false,
8666            verify_retried: false,
8667            e2e_deferred: true,
8668            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
8669            progressed: false,
8670            vote_split: false,
8671            reconsideration: Vec::new(),
8672            verdict: None,
8673        }];
8674
8675        let mut runner = Runner {
8676            state,
8677            roles: ResolvedRoles {
8678                implementers: Vec::new(),
8679                judges: Vec::new(),
8680                reviewers: Vec::new(),
8681                fixer: None,
8682                conductor: conductor(),
8683                implementer_roster: Vec::new(),
8684            },
8685            sem: Arc::new(Semaphore::new(1)),
8686            pause: Pause::new(),
8687            interrupt: Pause::new(),
8688        };
8689
8690        let shell = runner.state.config.shell();
8691        runner
8692            .stop_reviewing("round budget spent", &shell, &repo)
8693            .await
8694            .expect("stop_reviewing");
8695
8696        let last = runner.state.reviews.last().expect("round record");
8697        assert_eq!(
8698            last.e2e_status(),
8699            E2eStatus::ResourceBlocked,
8700            "the shared cache is still held; the attempt must read as blocked, not deferred or \
8701             failed: {last:?}"
8702        );
8703        assert_eq!(
8704            last.verified_head.as_deref(),
8705            Some(head.as_str()),
8706            "which commit this attempt targeted is known even though nothing finished checking \
8707             it"
8708        );
8709        let first_attempt_at = last
8710            .verified_at
8711            .expect("when this attempt ran is known too");
8712        assert_ne!(
8713            runner.state.status,
8714            RunStatus::Blocked,
8715            "contention is evidence about the machine, not the patch — it must not settle the \
8716             run as blocked: {:?}",
8717            runner.state.status
8718        );
8719        assert!(
8720            !runner
8721                .state
8722                .events
8723                .iter()
8724                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
8725            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
8726            runner.state.events
8727        );
8728
8729        // The cache is still held: a later reentry must retry the same
8730        // round's verification again — not leave it looking exactly as
8731        // untouched as the first blocked attempt, which is indistinguishable
8732        // from never having tried again at all.
8733        runner
8734            .stop_reviewing("round budget spent", &shell, &repo)
8735            .await
8736            .expect("stop_reviewing retry");
8737        assert_eq!(
8738            runner.state.reviews.len(),
8739            1,
8740            "no new round was started: {:?}",
8741            runner.state.reviews
8742        );
8743        let last = runner.state.reviews.last().expect("round record");
8744        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
8745        assert!(
8746            last.verified_at.expect("still known") > first_attempt_at,
8747            "a second reentry must be a fresh attempt, not a stale copy of the first"
8748        );
8749        assert_ne!(runner.state.status, RunStatus::Blocked);
8750
8751        held.release();
8752    }
8753
8754    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
8755    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
8756    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
8757    /// own top-of-function fast path (`review_conclusion`) correctly reads
8758    /// this shape as `None` rather than guessing `Blocked`, and the loop's
8759    /// own `for` range is empty once the round budget is spent, so
8760    /// `review_loop` must retry the check itself rather than silently doing
8761    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
8762    /// above exercises directly, but through `review_loop`'s own entry point
8763    /// this time, proving the wiring between the two rather than just the
8764    /// retry logic in isolation.
8765    #[tokio::test]
8766    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
8767        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8768        let home = crate::run::home();
8769
8770        let tmp = tempfile::tempdir().expect("tempdir");
8771        let repo = tmp.path().join("repo");
8772        std::fs::create_dir_all(&repo).unwrap();
8773        init_repo(&repo);
8774        let head = crate::git::rev_parse(&repo, "HEAD")
8775            .await
8776            .expect("rev-parse");
8777        let cache_dir = tmp.path().join("target");
8778
8779        let mut config = Config::default();
8780        config.verify.e2e = vec![format!(
8781            "CARGO_TARGET_DIR='{}' test -f README.md",
8782            cache_dir.display()
8783        )];
8784        config.graph.review_rounds = 1;
8785        config.graph.timeout_verify = Some(2);
8786
8787        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8788        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8789            .expect("no io error acquiring directly")
8790        {
8791            crate::cache::AcquireOutcome::Acquired(g) => g,
8792            crate::cache::AcquireOutcome::Busy(b) => {
8793                panic!("expected the direct acquire to win the lease first: {b:?}")
8794            }
8795        };
8796
8797        let mut state = RunState::new(
8798            repo.clone(),
8799            "main".to_owned(),
8800            head.clone(),
8801            "task".to_owned(),
8802            config,
8803        );
8804        state.candidates = vec![Candidate {
8805            index: 0,
8806            label: 'A',
8807            agent: "alpha".to_owned(),
8808            branch: "does-not-exist".to_owned(),
8809            worktree: repo.clone(),
8810            summary: String::new(),
8811            stat: String::new(),
8812            files: 0,
8813            commits: 0,
8814            empty: false,
8815            failed: None,
8816            verified_noop: None,
8817            duration_ms: 0,
8818            folded: false,
8819        }];
8820        state.tally = Some(Tally {
8821            first_choice: BTreeMap::from([('A', 1)]),
8822            borda: BTreeMap::new(),
8823            winner: 'A',
8824            rankings: 1,
8825            unanimous_initial: true,
8826            deliberated: false,
8827            changed_votes: 0,
8828            unanimous_final: true,
8829            tie_break: None,
8830            judges: 0,
8831            present: 0,
8832            quorum: 0,
8833            met_quorum: true,
8834            uncontested: Some("only candidate A produced a change".to_owned()),
8835        });
8836        // The exact shape a prior process's `stop_reviewing` would have left
8837        // on disk: the round budget's last round, a real attempt already
8838        // made and already resource-blocked.
8839        state.reviews = vec![ReviewRound {
8840            round: 1,
8841            head: head.clone(),
8842            verified_head: Some(head.clone()),
8843            verified_at: Some(jiff::Timestamp::now()),
8844            reviews: Vec::new(),
8845            e2e: vec![CommandOutcome {
8846                command: format!(
8847                    "CARGO_TARGET_DIR='{}' test -f README.md",
8848                    cache_dir.display()
8849                ),
8850                code: None,
8851                output_tail: "waiting for the shared build cache".to_owned(),
8852                duration_ms: 0,
8853                resource_blocked: true,
8854            }],
8855            fix: None,
8856            blocking: 1,
8857            answered: 1,
8858            expected: 1,
8859            clean: false,
8860            verify_retried: false,
8861            e2e_deferred: false,
8862            e2e_defer_reason: None,
8863            progressed: false,
8864            vote_split: false,
8865            reconsideration: Vec::new(),
8866            verdict: None,
8867        }];
8868
8869        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
8870        let mut runner = Runner {
8871            state,
8872            roles: ResolvedRoles {
8873                implementers: Vec::new(),
8874                judges: Vec::new(),
8875                reviewers: Vec::new(),
8876                fixer: None,
8877                conductor: conductor(),
8878                implementer_roster: Vec::new(),
8879            },
8880            sem: Arc::new(Semaphore::new(1)),
8881            pause: Pause::new(),
8882            interrupt: Pause::new(),
8883        };
8884
8885        // The lease is still held throughout, so this reentry's own retry is
8886        // also contended — proving `review_loop` actually tried again (not
8887        // that it happened to succeed) is what the timestamp comparison
8888        // below is for.
8889        runner.review_loop().await.expect("review_loop");
8890
8891        assert_eq!(
8892            runner.state.reviews.len(),
8893            1,
8894            "no new round was started on top of the unresolved one: {:?}",
8895            runner.state.reviews
8896        );
8897        let last = &runner.state.reviews[0];
8898        assert_eq!(
8899            last.e2e_status(),
8900            E2eStatus::ResourceBlocked,
8901            "still contended: {last:?}"
8902        );
8903        assert!(
8904            last.verified_at.expect("still known") > first_attempt_at,
8905            "review_loop must have actually retried the check, not left it exactly as found"
8906        );
8907        assert_ne!(
8908            runner.state.status,
8909            RunStatus::Blocked,
8910            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
8911            runner.state.status
8912        );
8913
8914        held.release();
8915    }
8916
8917    #[tokio::test]
8918    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
8919        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8920        let tmp = tempfile::tempdir().expect("tempdir");
8921        let repo = tmp.path().join("repo");
8922        std::fs::create_dir_all(&repo).unwrap();
8923        init_repo(&repo);
8924
8925        let mut config = Config::default();
8926        config.merge.mode = MergeMode::Pr;
8927        config.graph.land = true;
8928        config.graph.land_approval = false;
8929
8930        let mut state = RunState::new(
8931            repo.clone(),
8932            "main".to_owned(),
8933            "deadbeef".to_owned(),
8934            "task".to_owned(),
8935            config,
8936        );
8937        state.candidates = vec![Candidate {
8938            index: 0,
8939            label: 'A',
8940            agent: "alpha".to_owned(),
8941            branch: "does-not-exist".to_owned(),
8942            worktree: repo.clone(),
8943            summary: String::new(),
8944            stat: String::new(),
8945            files: 0,
8946            commits: 0,
8947            empty: false,
8948            failed: None,
8949            verified_noop: None,
8950            duration_ms: 0,
8951            folded: false,
8952        }];
8953        state.tally = Some(Tally {
8954            first_choice: BTreeMap::from([('A', 1)]),
8955            borda: BTreeMap::new(),
8956            winner: 'A',
8957            rankings: 1,
8958            unanimous_initial: true,
8959            deliberated: false,
8960            changed_votes: 0,
8961            unanimous_final: true,
8962            tie_break: None,
8963            judges: 0,
8964            present: 0,
8965            quorum: 0,
8966            met_quorum: true,
8967            uncontested: Some("only candidate A produced a change".to_owned()),
8968        });
8969        state.reviews = vec![ReviewRound {
8970            round: 1,
8971            head: "deadbeef".to_owned(),
8972            verified_head: None,
8973            verified_at: None,
8974            reviews: Vec::new(),
8975            e2e: Vec::new(),
8976            fix: None,
8977            blocking: 0,
8978            answered: 0,
8979            expected: 0,
8980            clean: true,
8981            verify_retried: false,
8982            e2e_deferred: false,
8983            e2e_defer_reason: None,
8984            progressed: false,
8985            vote_split: false,
8986            reconsideration: Vec::new(),
8987            verdict: None,
8988        }];
8989        state.gate = vec![CommandOutcome {
8990            command: "test".to_owned(),
8991            code: Some(0),
8992            output_tail: String::new(),
8993            duration_ms: 0,
8994            resource_blocked: false,
8995        }];
8996        state.gate_ran = true;
8997        // A first pass through `merge` already pushed and opened this pull
8998        // request; `status` is `Landing` because a previous call into `land`
8999        // parked or was interrupted before it reached a terminal outcome.
9000        state.status = RunStatus::Landing;
9001        state.merge = Some(MergeOutcome {
9002            mode: MergeMode::Pr,
9003            ok: true,
9004            detail: "https://example.invalid/x/y/pull/1".to_owned(),
9005        });
9006
9007        // The Landing-resume shortcut calls `run_land` directly rather than
9008        // through `merge`, which is exactly the call site that used to skip
9009        // `settle_questions` - see the fixture below.
9010        ask_test_home();
9011        let store = ask::Questions::open();
9012        let q = ask_open_question(&store, &state.id);
9013
9014        let mut runner = Runner {
9015            state,
9016            roles: ResolvedRoles {
9017                implementers: Vec::new(),
9018                judges: Vec::new(),
9019                reviewers: Vec::new(),
9020                fixer: None,
9021                conductor: conductor(),
9022                implementer_roster: Vec::new(),
9023            },
9024            sem: Arc::new(Semaphore::new(1)),
9025            pause: Pause::new(),
9026            interrupt: Pause::new(),
9027        };
9028
9029        // `execute`, not `merge` directly: the Landing-resume shortcut lives
9030        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
9031        // exactly because `review_loop` would otherwise clobber the marker
9032        // first.
9033        runner.execute().await.expect("execute");
9034
9035        assert_eq!(
9036            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
9037            Some("https://example.invalid/x/y/pull/1"),
9038            "reentry must not push again or open a second pull request over the \
9039             one `land` is already watching"
9040        );
9041        assert_ne!(
9042            runner.state.status,
9043            RunStatus::Landing,
9044            "land could not actually reach the fake pull request, so it must \
9045             have given up rather than left the run silently parked forever"
9046        );
9047        // `land` could not reach the fake pull request, so it gave up into
9048        // `Blocked` - still resumable, so the question must not have been
9049        // swept just because this branch now also calls `settle_questions`.
9050        assert_eq!(runner.state.status, RunStatus::Blocked);
9051        assert!(
9052            store.get(&q.id).unwrap().status.open(),
9053            "Blocked is still alive; settle_questions must have been a no-op here"
9054        );
9055    }
9056
9057    fn state_with_round(round: ReviewRound) -> RunState {
9058        let mut s = RunState::new(
9059            PathBuf::from("/repo"),
9060            "main".to_owned(),
9061            "abc1234".to_owned(),
9062            "add retries".to_owned(),
9063            Config::default(),
9064        );
9065        s.reviews = vec![round];
9066        s
9067    }
9068
9069    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
9070        crate::verdict::Finding {
9071            id: id.to_owned(),
9072            severity,
9073            file: None,
9074            line: None,
9075            title: title.to_owned(),
9076            detail: String::new(),
9077        }
9078    }
9079
9080    #[test]
9081    fn pr_body_names_open_findings_and_declined_ones() {
9082        let round = ReviewRound {
9083            round: 2,
9084            head: "deadbee".to_owned(),
9085            verified_head: None,
9086            verified_at: None,
9087            reviews: vec![ReviewRecord {
9088                attempts: 0,
9089                reviewer: 1,
9090                agent: "alpha".to_owned(),
9091                summary: String::new(),
9092                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
9093                vote: None,
9094                failed: None,
9095                duration_ms: 0,
9096            }],
9097            e2e: vec![CommandOutcome {
9098                command: "cargo test".to_owned(),
9099                code: Some(0),
9100                output_tail: String::new(),
9101                duration_ms: 0,
9102                resource_blocked: false,
9103            }],
9104            verify_retried: false,
9105            e2e_deferred: false,
9106            e2e_defer_reason: None,
9107            fix: Some(FixRecord {
9108                agent: "alpha".to_owned(),
9109                addressed: Vec::new(),
9110                rejected: vec![crate::verdict::Rejection {
9111                    id: "R1-1-1".to_owned(),
9112                    why: "not reachable from any caller".to_owned(),
9113                }],
9114                notes: String::new(),
9115                committed: true,
9116                failed: None,
9117                duration_ms: 0,
9118                continuation: None,
9119            }),
9120            blocking: 0,
9121            answered: 1,
9122            expected: 1,
9123            clean: false,
9124            progressed: true,
9125            vote_split: false,
9126            reconsideration: Vec::new(),
9127            verdict: None,
9128        };
9129        let state = state_with_round(round);
9130        let body = pr_message(&state, 'A').body;
9131
9132        assert!(body.contains("add retries"), "the task must still be there");
9133        assert!(body.contains("R2-1-1"), "{body}");
9134        assert!(body.contains("unused import"), "{body}");
9135        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
9136        assert!(
9137            body.contains("not reachable from any caller"),
9138            "the reason it was declined: {body}"
9139        );
9140    }
9141
9142    #[test]
9143    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
9144        let round = ReviewRound {
9145            round: 1,
9146            head: "deadbee".to_owned(),
9147            verified_head: None,
9148            verified_at: None,
9149            reviews: vec![ReviewRecord {
9150                attempts: 0,
9151                reviewer: 1,
9152                agent: "alpha".to_owned(),
9153                summary: String::new(),
9154                findings: Vec::new(),
9155                vote: None,
9156                failed: None,
9157                duration_ms: 0,
9158            }],
9159            e2e: Vec::new(),
9160            verify_retried: false,
9161            e2e_deferred: false,
9162            e2e_defer_reason: None,
9163            fix: None,
9164            blocking: 0,
9165            answered: 1,
9166            expected: 1,
9167            clean: true,
9168            progressed: false,
9169            vote_split: false,
9170            reconsideration: Vec::new(),
9171            verdict: None,
9172        };
9173        let state = state_with_round(round);
9174        let body = pr_message(&state, 'A').body;
9175        assert!(!body.contains("Open review findings"), "{body}");
9176        assert!(!body.contains("Declined"), "{body}");
9177    }
9178
9179    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
9180        let mut state = RunState::new(
9181            PathBuf::from("/repo"),
9182            "main".to_owned(),
9183            "abc1234".to_owned(),
9184            instruction.to_owned(),
9185            Config::default(),
9186        );
9187        state.candidates.push(Candidate {
9188            index: 0,
9189            label: 'A',
9190            agent: "alpha".to_owned(),
9191            branch: "magi/x/A".to_owned(),
9192            worktree: PathBuf::from("/wt"),
9193            summary: summary.to_owned(),
9194            stat: String::new(),
9195            files: 1,
9196            commits: 1,
9197            empty: false,
9198            failed: None,
9199            verified_noop: None,
9200            folded: false,
9201            duration_ms: 0,
9202        });
9203        state
9204    }
9205
9206    #[test]
9207    fn pr_message_describes_the_change_not_the_task() {
9208        let state = state_with_summary(
9209            "今回やってほしいこと: results projector を直す",
9210            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
9211        );
9212        let m = pr_message(&state, 'A');
9213        assert_eq!(m.title, "fix(web): batch the runs list reads");
9214        assert!(
9215            m.body.starts_with("## Summary\n\n- reads run.json once"),
9216            "{}",
9217            m.body
9218        );
9219        assert!(!m.body.contains("TITLE:"), "{}", m.body);
9220        let task_at = m.body.find("今回やってほしいこと").unwrap();
9221        let details_at = m.body.find("<details>").unwrap();
9222        assert!(
9223            details_at < task_at,
9224            "the task lives inside <details>: {}",
9225            m.body
9226        );
9227        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
9228        assert!(m.body.contains("magi:candidate-a"));
9229    }
9230
9231    #[test]
9232    fn pr_message_falls_back_to_the_task_without_a_title_line() {
9233        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
9234        let m = pr_message(&state, 'A');
9235        assert_eq!(m.title, "add retries");
9236        assert!(
9237            m.body.contains("## Summary\n\n- did some things"),
9238            "{}",
9239            m.body
9240        );
9241
9242        let none = RunState::new(
9243            PathBuf::from("/repo"),
9244            "main".to_owned(),
9245            "abc1234".to_owned(),
9246            "add retries".to_owned(),
9247            Config::default(),
9248        );
9249        let m = pr_message(&none, 'A');
9250        assert_eq!(m.title, "add retries");
9251        assert!(!m.body.contains("## Summary"), "{}", m.body);
9252    }
9253
9254    #[test]
9255    fn pr_message_refuses_the_candidate_commit_subject() {
9256        for bad in [
9257            "TITLE: magi: candidate A (uncommitted work)",
9258            "TITLE: chore: stuff (uncommitted work)",
9259            "TITLE:   ",
9260        ] {
9261            let state = state_with_summary("add retries", bad);
9262            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
9263        }
9264    }
9265
9266    #[test]
9267    fn pr_message_bounds_a_very_long_task_and_title() {
9268        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
9269        let state = state_with_summary(&long, "- nothing");
9270        let m = pr_message(&state, 'A');
9271        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
9272        assert!(!m.title.contains('\n'));
9273
9274        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
9275        let m = pr_message(&state, 'A');
9276        assert!(m.title.starts_with("feat: "));
9277        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
9278        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
9279    }
9280
9281    #[test]
9282    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
9283        // What magi itself writes stays English under any configured language,
9284        // so a future localisation of these headings fails here. (The agents'
9285        // own text is held to English by the prompt only; magi cannot check it.)
9286        let mut state = state_with_summary(
9287            "add retries",
9288            "TITLE: fix(web): batch reads\n- reads run.json once",
9289        );
9290        state.config.graph.language = "ja".to_owned();
9291        let m = pr_message(&state, 'A');
9292        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
9293
9294        // The task is the operator's own text: it goes in untouched, and the
9295        // fallback title (no summary) may be in its language too.
9296        let task = "今回やってほしいこと: results projector を直す";
9297        let mut state = state_with_summary(task, "- no title line");
9298        state.config.graph.language = "ja".to_owned();
9299        let m = pr_message(&state, 'A');
9300        assert_eq!(
9301            m.title,
9302            format!("chore: land candidate A of run {}", state.id)
9303        );
9304        assert!(
9305            m.body.contains(&format!(
9306                "<summary>Original task</summary>\n\n{task}\n\n</details>"
9307            )),
9308            "{}",
9309            m.body
9310        );
9311    }
9312
9313    #[test]
9314    fn pr_message_scrubs_home_paths_and_addresses() {
9315        let state = state_with_summary(
9316            "fix it in /Users/someone/src/x",
9317            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
9318        );
9319        let m = pr_message(&state, 'A');
9320        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
9321            assert!(!m.body.contains(leak), "{}", m.body);
9322        }
9323        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
9324    }
9325
9326    #[test]
9327    fn pr_message_survives_a_task_that_closes_details() {
9328        let state = state_with_summary("a </details> b", "TITLE: fix: x");
9329        let m = pr_message(&state, 'A');
9330        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
9331    }
9332
9333    #[test]
9334    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
9335        let cmd = manual_merge_command(
9336            MergeStyle::Squash,
9337            Path::new("/repo"),
9338            "b",
9339            "fix: \"quoted\" $(x) `y`\n\nbody",
9340        );
9341        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
9342    }
9343
9344    #[test]
9345    fn manual_merge_command_matches_the_configured_style() {
9346        let repo = Path::new("/repo");
9347        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
9348
9349        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
9350        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
9351
9352        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
9353        assert_eq!(
9354            squash,
9355            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
9356             \"Merge magi run 0832 (candidate A)\""
9357        );
9358
9359        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
9360        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
9361    }
9362
9363    #[test]
9364    fn a_nudge_gets_a_quarter_of_the_budget() {
9365        // The judge and implement budgets magi ships with.
9366        assert_eq!(retry_budget(secs(1200), true), secs(300));
9367        assert_eq!(retry_budget(secs(3600), true), secs(900));
9368    }
9369
9370    #[test]
9371    fn a_resent_prompt_keeps_the_whole_budget() {
9372        // The seat kept no context, so the retry is the original job again and
9373        // shortening it would only guarantee a second failure.
9374        assert_eq!(retry_budget(secs(1200), false), secs(1200));
9375        assert_eq!(retry_budget(secs(60), false), secs(60));
9376    }
9377
9378    #[test]
9379    fn the_floor_never_exceeds_the_original_budget() {
9380        // A short configured timeout must not be *raised* by the floor: the
9381        // operator asked for a bound, and a retry may not outlast the attempt
9382        // it is retrying.
9383        assert_eq!(retry_budget(secs(60), true), secs(60));
9384        assert_eq!(retry_budget(secs(480), true), secs(120));
9385        assert_eq!(retry_budget(secs(0), true), secs(0));
9386    }
9387
9388    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
9389        agent::CommandEvidence {
9390            id: "item1".to_owned(),
9391            description: "cargo test".to_owned(),
9392            exit_code,
9393            result_summary: String::new(),
9394            source: "codex".to_owned(),
9395        }
9396    }
9397
9398    #[test]
9399    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
9400        // No evidence is not the same fact as unconfirmed evidence: a
9401        // backend with no adapter, or a reply that ran no commands at all,
9402        // must not be misread as carrying a dangling job.
9403        assert!(!has_unconfirmed_command(&[]));
9404    }
9405
9406    #[test]
9407    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
9408        // Deliberately not a check on the exit code's *value*: a fixer
9409        // legitimately runs something that fails mid-iteration before it
9410        // succeeds, and that must never by itself reopen a valid report.
9411        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
9412        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
9413        assert!(!has_unconfirmed_command(&[
9414            evidence(Some(0)),
9415            evidence(Some(101))
9416        ]));
9417    }
9418
9419    #[test]
9420    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
9421        assert!(has_unconfirmed_command(&[
9422            evidence(Some(0)),
9423            evidence(None)
9424        ]));
9425    }
9426
9427    #[test]
9428    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
9429        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9430        assert_eq!(
9431            verified_noop_claim(true, &[], text).as_deref(),
9432            Some("already fixed by b32cfc4, on main.")
9433        );
9434    }
9435
9436    #[test]
9437    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
9438        // A timeout or a bad exit code reads as the ordinary loss it is,
9439        // whatever the reply's own prose claims.
9440        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9441        assert!(verified_noop_claim(false, &[], text).is_none());
9442    }
9443
9444    #[test]
9445    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
9446        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9447        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
9448        // A confirmed command alongside the marker is fine.
9449        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
9450    }
9451
9452    #[test]
9453    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
9454        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
9455    }
9456
9457    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
9458    /// pair, in order, labelled A, B, C, ...
9459    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
9460        runner.state.candidates = shape
9461            .iter()
9462            .enumerate()
9463            .map(|(i, &(empty, verified))| Candidate {
9464                index: i,
9465                label: (b'A' + i as u8) as char,
9466                agent: "sonnet".to_owned(),
9467                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
9468                worktree: PathBuf::from(format!("/wt/{i}")),
9469                summary: String::new(),
9470                stat: String::new(),
9471                files: 0,
9472                commits: 0,
9473                empty,
9474                failed: None,
9475                verified_noop: verified.map(str::to_owned),
9476                duration_ms: 0,
9477                folded: false,
9478            })
9479            .collect();
9480    }
9481
9482    #[test]
9483    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
9484        ask_test_home();
9485        let mut runner = runner_at(RunStatus::Implementing);
9486        set_candidates(
9487            &mut runner,
9488            &[
9489                (true, Some("already on main at b32cfc4")),
9490                (true, Some("same fix, see the existing test")),
9491            ],
9492        );
9493
9494        runner
9495            .after_implement()
9496            .expect("a verified no-op is not an error");
9497
9498        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
9499    }
9500
9501    #[test]
9502    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
9503        ask_test_home();
9504        let mut runner = runner_at(RunStatus::Implementing);
9505        // Candidate A declares a verified no-op; candidate B simply wrote
9506        // nothing and said nothing about why. One candidate's claim is not
9507        // the whole run's agreement.
9508        set_candidates(
9509            &mut runner,
9510            &[(true, Some("already on main at b32cfc4")), (true, None)],
9511        );
9512
9513        let err = runner
9514            .after_implement()
9515            .expect_err("an unverified empty candidate must still fail the run");
9516
9517        assert!(
9518            err.to_string().contains("no candidate produced a change"),
9519            "{err}"
9520        );
9521        assert_eq!(runner.state.status, RunStatus::Failed);
9522    }
9523
9524    #[test]
9525    fn after_implement_still_fails_an_ordinary_all_empty_run() {
9526        ask_test_home();
9527        let mut runner = runner_at(RunStatus::Implementing);
9528        set_candidates(&mut runner, &[(true, None), (true, None)]);
9529
9530        let err = runner
9531            .after_implement()
9532            .expect_err("no candidate declared anything; this is an ordinary failure");
9533
9534        assert!(
9535            err.to_string().contains("no candidate produced a change"),
9536            "{err}"
9537        );
9538        assert_eq!(runner.state.status, RunStatus::Failed);
9539    }
9540}