Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::run::{
46    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
47    DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, GateFixRecord, JobRecord, JobStatus,
48    Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, QuotaLoss,
49    ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord, tail,
50    write_artifact,
51};
52use crate::verdict::{
53    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
54    ReviewVote, Severity,
55};
56
57/// How much verification output is kept and fed back to the fixer.
58const OUTPUT_TAIL: usize = 8_000;
59
60/// Bytes of a failing command's output kept in an event, so the reason a run
61/// stopped is readable from the report without opening `run.json`.
62const EVENT_OUTPUT_TAIL: usize = 2_000;
63
64/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
65/// command's pid before releasing the build cache's lease.
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
67
68/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
69/// command's pid to actually exit before giving up and releasing anyway.
70///
71/// A timeout means the process was asked to die (`kill_on_drop`,
72/// `start_kill`), not that it already has — on Windows in particular that can
73/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
74/// the instant the command returns would let the very next acquirer (this
75/// run's own next round, another run's verification, the janitor's prune)
76/// start touching the same directory while it might still be writing to it,
77/// so this polls the actual pid — real confirmation, not a fixed guess —
78/// until it is gone or this ceiling is reached. It is still not full
79/// process-tree reaping: a grandchild the timed-out process spawned and that
80/// outlives it independently is invisible to a pid check, and continuing to
81/// observe and collect *that* stays a different piece of work with its own
82/// owner. Set generously because the common case returns early the moment
83/// the pid is confirmed gone, not because every timeout pays this in full.
84const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
85
86/// Consecutive review rounds with no tree progress (see
87/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
88/// instead of spending the rest of the round budget.
89///
90/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
91/// legitimately finds nothing left to change (its previous round's fix already
92/// covered it, and this round's reviewers re-raised only nits) looks the same
93/// as one that is spinning, for exactly one round. Two in a row is where the
94/// two stop being distinguishable, and a review round on this workload has
95/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
96/// third attempt at a tree that has not moved twice running is pure cost.
97/// This does not touch `review_rounds` itself, which stays the operator's
98/// call.
99pub(crate) const STAGNANT_LIMIT: usize = 2;
100
101/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
102/// a base that moved before giving up and leaving the run `Blocked` for a
103/// person.
104///
105/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
106/// that keeps moving faster than a run can catch it is not something more
107/// rebasing fixes, it is a person's call. Not the same *number as*
108/// `land_rounds` - this budget is spent before a pull request exists, land's
109/// after - but bounded for the identical reason, so it uses the same
110/// default. Counted across both call sites in [`Runner::finish_after_tally`]
111/// (once before review, once before the gate), because either one finding
112/// the base still moving is the same signal.
113const BASE_SYNC_ROUNDS: usize = 4;
114
115/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
116/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
117/// reply held no [`FixReport`].
118///
119/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
120/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
121/// "I'll pause here until the `cargo make check` background run reports
122/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
123/// No `FixReport` was ever collected from that seat, and the run moved on to
124/// the next review round regardless.
125///
126/// Bounded independently of `review_rounds` and `graph.retries`: this
127/// recovers one seat's missing report mid-round, not a new round of review or
128/// an ordinary parse retry, and must not itself become the unbounded wait the
129/// rest of this module exists to avoid.
130const MAX_FIX_CONTINUATIONS: usize = 2;
131
132/// One queued agent invocation.
133///
134/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
135/// CLI hung up on its own stream is asked again from the same job rather than
136/// rebuilt from scratch. See [`Runner::resume_undelivered`].
137#[derive(Clone)]
138struct SeatJob {
139    spec: AgentSpec,
140    seat: SeatState,
141    cwd: PathBuf,
142    prompt: String,
143    timeout: Duration,
144    allow_write: bool,
145    sessions: bool,
146    artifacts: PathBuf,
147    stem: String,
148}
149
150/// How the graph reads one agent invocation.
151///
152/// Quota is split out from an ordinary failure on purpose: a rate-limited call
153/// is known to fail again if retried now, so the retry loop must not spend an
154/// attempt on it. `Dropped` is split out for the opposite reason: unlike
155/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
156/// error JSON, never the agent's answer — a caller that matched only
157/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
158/// left to read that JSON as if it were usable output. `resume_undelivered`
159/// is the only caller that acts on it; everywhere else it is reported like an
160/// ordinary failure.
161enum AgentOutcome {
162    /// A usable output.
163    Ok(AgentOutput),
164    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
165    Quota(AgentOutput),
166    /// The CLI hung up on its own stream after billed work. See
167    /// [`agent::AgentOutput::work_undelivered`].
168    Dropped(AgentOutput),
169    /// Any other failure: a timeout, a bad exit code, an empty reply.
170    Failed(String),
171}
172
173/// A request to park the run at its next node boundary.
174///
175/// Cloning is how the request travels: the loop keeps one handle and hands a
176/// clone to each [`Runner`], and every clone points at the same flag. There
177/// is no channel because there is nothing to send - the only message is
178/// "park", it is idempotent, and a flag cannot be missed by a receiver that
179/// was not listening yet.
180///
181/// The boundary is what makes this cheap. Every node writes the run's state
182/// before the next one starts, and every node skips what is already recorded:
183/// `prep` returns early once candidates exist, `implement` asks only the seats
184/// with nothing on disk, `judge` returns early once judgements exist. So a
185/// parked run resumes into exactly the node it stopped before, and no agent
186/// work is thrown away. Killing the process mid-node, by contrast, loses
187/// whatever the seats in flight had not yet written - which for an implement
188/// wave is an hour of paid work.
189///
190/// A [`Runner`] watches two independent handles of this type - see
191/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
192/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
193/// clone covering the whole daemon's lifetime and is never asked to un-park,
194/// which is correct exactly because nothing is dispatched after it fires.
195/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
196/// that parks for an interrupted task must go on to run other tasks
197/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
198/// reusing the daemon-wide one.
199#[derive(Debug, Clone, Default)]
200pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
201
202impl Pause {
203    /// A pause nobody has asked for yet.
204    #[must_use]
205    pub fn new() -> Self {
206        Self::default()
207    }
208
209    /// Ask the run to park at its next node boundary. Idempotent.
210    pub fn park(&self) {
211        self.0.store(true, Ordering::SeqCst);
212    }
213
214    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
215    /// fold into the run's own `park` event - so an operator reading the run
216    /// later knows this was a deliberate interrupt rather than a shutdown or
217    /// a binary swap. The first reason recorded wins; a park already in
218    /// flight is not relabelled by a second, unrelated request.
219    pub fn park_because(&self, reason: impl Into<String>) {
220        let mut reason_guard = self
221            .1
222            .lock()
223            .unwrap_or_else(std::sync::PoisonError::into_inner);
224        if reason_guard.is_none() {
225            *reason_guard = Some(reason.into());
226        }
227        drop(reason_guard);
228        self.park();
229    }
230
231    /// Has a park been asked for?
232    #[must_use]
233    pub fn parked(&self) -> bool {
234        self.0.load(Ordering::SeqCst)
235    }
236
237    /// Why the park was asked for, when the caller used [`Pause::park_because`].
238    #[must_use]
239    pub fn reason(&self) -> Option<String> {
240        self.1
241            .lock()
242            .unwrap_or_else(std::sync::PoisonError::into_inner)
243            .clone()
244    }
245}
246
247/// Drives one run.
248pub struct Runner {
249    /// Run state; public so the CLI can report on it.
250    pub state: RunState,
251    roles: ResolvedRoles,
252    sem: Arc<Semaphore>,
253    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
254    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
255    /// this is never the same handle as `interrupt`.
256    pause: Pause,
257    /// Set when `magi serve`'s interrupt scheduler wants this specific run
258    /// parked at its next node boundary, to let a task marked
259    /// [`crate::queue::Task::interrupt`] run alone before this one carries
260    /// on. Unlike `pause`, a fresh, unshared handle per run - see
261    /// [`Runner::watch_interrupt`].
262    interrupt: Pause,
263}
264
265/// The commit a run branches from: the base branch as the remote has it.
266///
267/// Two failures this replaces. A run used to branch off `HEAD` and so refused
268/// to start on a dirty tree, which made `magi serve` decline every task for as
269/// long as the operator had work in progress - most of the time. Branching off
270/// the *local* base branch fixed that and introduced a worse one: `land` merges
271/// the winner on GitHub, nothing updates the local ref, and the next run
272/// branches off a base missing everything the previous runs landed. Two tasks
273/// in a row from a phone would have had the second silently re-implementing
274/// against stale code and opening a pull request that reverted the first.
275///
276/// Only refs move here - no checkout, no local branch, no merge - so it is safe
277/// with uncommitted work in the tree. A machine with no network still starts:
278/// the fetch may fail and the local tip is used with a warning, because
279/// refusing to run offline is a worse failure than running against a base the
280/// operator can see for themselves.
281///
282/// One function, called by both entry points. Two answers to "where does a run
283/// branch from" is the kind of drift nobody notices until a diff is wrong.
284/// Bring the local `branch` in line with `<remote>/<branch>` before a review
285/// checks it out.
286///
287/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
288/// anything other than plain `git push` from this checkout (a jj colocated
289/// workspace, another clone) moves only the remote-tracking ref - so the local
290/// one can be a stale placeholder. It moves only when local is behind the remote or is an
291/// empty placeholder that diverged from it; unpushed local work is kept, and a real
292/// divergence is refused rather than guessed at.
293async fn sync_review_branch(repo: &Path, branch: &str, remote: &str) -> Result<()> {
294    let tracking = format!("{remote}/{branch}");
295    let fetched = git::fetch(repo, remote, branch).await;
296    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
297    let local_exists = git::branch_exists(repo, branch).await?;
298    if !fresh {
299        if !local_exists {
300            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
301        }
302        tracing::warn!(
303            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
304        );
305        return Ok(());
306    }
307    let remote_sha = git::rev_parse(repo, &tracking).await?;
308    if !local_exists {
309        git::git(repo, &["branch", branch, &tracking]).await?;
310        return Ok(());
311    }
312    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
313    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
314        return Ok(());
315    }
316    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
317        // Diverged. A local tip that adds nothing over the fork point is a
318        // placeholder the remote's work replaced (a jj rewrite of the same
319        // change); anything else is local work we must not discard.
320        let mb = git::git_raw(repo, &["merge-base", &local_sha, &remote_sha]).await?;
321        let placeholder = mb.ok()
322            && git::git_raw(repo, &["diff", "--quiet", &mb.stdout, &local_sha])
323                .await?
324                .ok();
325        if !placeholder {
326            bail!(
327                "local `{branch}` ({}) and {tracking} ({}) have diverged, so it is unclear \
328                 which one to review; reconcile them, e.g. `git branch -f {branch} {tracking}` \
329                 to review the pushed work, or push the local branch first",
330                short(&local_sha),
331                short(&remote_sha)
332            );
333        }
334    }
335    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
336    if !out.ok() {
337        bail!(
338            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
339            short(&local_sha),
340            short(&remote_sha),
341            out.stderr
342        );
343    }
344    tracing::warn!(
345        "local `{branch}` was stale: fast-forwarded {} -> {}",
346        short(&local_sha),
347        short(&remote_sha)
348    );
349    Ok(())
350}
351
352async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
353    let tracking = format!("{remote}/{base_branch}");
354    let fetched = git::fetch(repo, remote, base_branch).await;
355    if let Ok(out) = &fetched
356        && out.ok()
357        && git::rev_exists(repo, &tracking).await
358    {
359        return git::rev_parse(repo, &tracking).await;
360    }
361    let why = match &fetched {
362        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
363        Ok(_) => format!("{remote} has no {base_branch}"),
364        Err(e) => e.to_string(),
365    };
366    tracing::warn!(
367        "could not read {tracking} ({why}); branching off the local \
368         {base_branch} instead, which may be behind"
369    );
370    git::rev_parse(repo, base_branch).await.with_context(|| {
371        format!(
372            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
373             branch that exists"
374        )
375    })
376}
377
378/// Exclusive claim on one run's `magi fix` step, released on drop — including
379/// on an early return or a panic.
380///
381/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
382/// manual `magi fix` invocations against the same run are otherwise
383/// invisible to each other and would race to remove and recreate the same
384/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
385/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
386/// which is only ever reclaimed later, out of band, by
387/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
388/// `magi fix` invocation is not necessarily running under either of those, so
389/// nothing would ever sweep a lock a killed or crashed process left behind.
390/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
391/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
392/// lease use: an unreadable or unparsable pid, or a liveness query the
393/// platform cannot answer, reads as alive and the lock is left in place.
394struct FixClaim {
395    path: PathBuf,
396}
397
398impl FixClaim {
399    fn acquire(dir: &Path) -> Result<Self> {
400        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
401        let path = dir.join("fix.lock");
402        match Self::create(&path) {
403            Ok(claim) => Ok(claim),
404            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
405                if Self::reclaim_if_dead(&path) {
406                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
407                } else {
408                    bail!(
409                        "another `magi fix` is already running for this run ({} exists)",
410                        path.display()
411                    )
412                }
413            }
414            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
415        }
416    }
417
418    fn create(path: &Path) -> std::io::Result<Self> {
419        let mut f = std::fs::OpenOptions::new()
420            .write(true)
421            .create_new(true)
422            .open(path)?;
423        use std::io::Write as _;
424        // Read back by `reclaim_if_dead` on a later, stuck invocation.
425        writeln!(f, "{}", std::process::id())?;
426        Ok(Self {
427            path: path.to_owned(),
428        })
429    }
430
431    /// True if the lock named a process confirmed dead, in which case it was
432    /// also removed. Never true on an unreadable file, an unparsable pid, or
433    /// a liveness query the platform cannot answer — see this type's own doc.
434    fn reclaim_if_dead(path: &Path) -> bool {
435        let dead = std::fs::read_to_string(path)
436            .ok()
437            .and_then(|body| body.trim().parse::<u32>().ok())
438            .is_some_and(|pid| !crate::proc::pid_alive(pid));
439        dead && std::fs::remove_file(path).is_ok()
440    }
441}
442
443impl Drop for FixClaim {
444    fn drop(&mut self) {
445        let _ = std::fs::remove_file(&self.path);
446    }
447}
448
449impl Runner {
450    /// Start a fresh run against `repo`.
451    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
452        let repo = git::toplevel(repo).await?;
453        let missing = agent::missing_programs(&config.agents);
454        if !missing.is_empty() {
455            bail!(
456                "these agent programs are not on PATH: {}. Fix the roster in \
457                 magi.toml or install them.",
458                missing.join(", ")
459            );
460        }
461        let base_branch = match config.merge.base.clone() {
462            Some(b) => b,
463            None => git::current_branch(&repo)
464                .await?
465                .context("HEAD is detached; set [merge] base in magi.toml")?,
466        };
467        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
468        // Still worth saying out loud. The operator's uncommitted work is not
469        // part of this run, and someone watching a candidate fail to use a
470        // change they just made deserves to know why.
471        if !git::is_clean(&repo).await? {
472            tracing::warn!(
473                "{} has uncommitted changes; they are not part of this run, \
474                 which branches off {base_branch} ({})",
475                repo.display(),
476                &base_commit[..base_commit.len().min(8)]
477            );
478        }
479        let roles = config.resolve_roles()?;
480        let max_parallel = config.graph.max_parallel.max(1);
481        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
482        state.event("start", format!("run {} created", state.id));
483        state.save()?;
484        Ok(Self {
485            state,
486            roles,
487            sem: Arc::new(Semaphore::new(max_parallel)),
488            pause: Pause::new(),
489            interrupt: Pause::new(),
490        })
491    }
492
493    /// Open a review-only run against work that already exists on `branch`.
494    ///
495    /// The expensive half of the graph is the implement wave — measured at
496    /// 111 and 134 internal tool-loop turns on this repository, against a
497    /// handful for a judge or a reviewer. The cheap half is worth running on
498    /// hand-written work too, and there was no way to reach it.
499    ///
500    /// No new state and no schema change are needed: a run with **one** viable
501    /// candidate and a tally already decided degrades `execute` to exactly
502    /// review → gate → merge, because `judge` skips a single-candidate field,
503    /// `deliberate` has fewer than two first choices to reconcile, `vote`
504    /// returns early, `tally` is already present and `fold_losers` has no
505    /// losers. Resuming such a run therefore does the right thing as well.
506    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
507        Self::review_taking_over(repo, branch, config, None).await
508    }
509
510    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
511    /// at the same task still has `branch` checked out, its worktree is
512    /// released first if that is safe (see [`crate::handover`]), and the
513    /// review refuses with the reason if it is not. `None` is a hand-run
514    /// review and behaves exactly as [`Runner::review`] always did.
515    pub async fn review_taking_over(
516        repo: &Path,
517        branch: &str,
518        config: Config,
519        takeover: Option<crate::handover::Takeover>,
520    ) -> Result<Self> {
521        let repo = git::toplevel(repo).await?;
522        let missing = agent::missing_programs(&config.agents);
523        if !missing.is_empty() {
524            bail!(
525                "these agent programs are not on PATH: {}. Fix the roster in \
526                 magi.toml or install them.",
527                missing.join(", ")
528            );
529        }
530        let base_branch = match config.merge.base.clone() {
531            Some(b) => b,
532            None => git::current_branch(&repo)
533                .await?
534                .context("HEAD is detached; set [merge] base in magi.toml")?,
535        };
536        if base_branch == branch {
537            bail!("`{branch}` is the base branch; there is nothing to review against");
538        }
539        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
540
541        let roles = config.resolve_roles()?;
542        let max_parallel = config.graph.max_parallel.max(1);
543        let mut state = RunState::new(
544            repo.clone(),
545            base_branch,
546            base_commit.clone(),
547            String::new(),
548            config,
549        );
550
551        // Released before anything else touches the branch: a stale local
552        // branch is moved with `git branch -f`, which git refuses while an
553        // earlier attempt's worktree still has it checked out. Everything
554        // after this point that can fail puts the old run back.
555        let released = match &takeover {
556            Some(takeover) => crate::handover::release(&repo, branch, &state.id, takeover).await?,
557            None => None,
558        };
559        if let Some(released) = &released {
560            state.event(
561                "release",
562                format!(
563                    "took `{branch}` over from run {}: its worktree was released",
564                    crate::run::short_of(&released.old_id)
565                ),
566            );
567        }
568        let opened =
569            Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
570        if opened.is_err()
571            && let Some(released) = &released
572        {
573            released.restore(&repo, branch).await;
574        }
575        opened
576    }
577
578    /// The half of [`Runner::review_taking_over`] that can fail after an
579    /// earlier attempt's worktree was released.
580    async fn open_review(
581        repo: &Path,
582        branch: &str,
583        mut state: RunState,
584        roles: ResolvedRoles,
585        max_parallel: usize,
586        base_commit: String,
587    ) -> Result<Self> {
588        sync_review_branch(repo, branch, &state.config.merge.remote).await?;
589        // The commit subjects are the closest thing to a task statement that
590        // existing work carries, and the reviewers are told as much.
591        let log = git::log_oneline(repo, &base_commit, branch)
592            .await
593            .unwrap_or_default();
594        let instruction = format!(
595            "Review the work already on branch `{branch}`. There is no task \
596             statement: what the change claims to do is whatever its commits \
597             say.\n\n{}",
598            if log.trim().is_empty() {
599                "(no commit messages)"
600            } else {
601                log.trim()
602            }
603        );
604        state.instruction = instruction;
605
606        // An attached worktree, so the fixer's commits land on the branch under
607        // review rather than on a detached head nobody will look at again.
608        let worktree = state.worktree_root().join("under-review");
609        if let Some(parent) = worktree.parent() {
610            tokio::fs::create_dir_all(parent).await.ok();
611        }
612        let path = worktree.to_string_lossy().to_string();
613        git::git(repo, &["worktree", "add", &path, branch])
614            .await
615            .with_context(|| {
616                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
617            })?;
618
619        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
620            .await
621            .unwrap_or(0);
622        if commits == 0 {
623            git::worktree_remove(repo, &worktree).await.ok();
624            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
625        }
626        let files = git::changed_files(&worktree, &base_commit, "HEAD")
627            .await
628            .map(|f| f.len())
629            .unwrap_or(0);
630        if files == 0
631            && let (Ok(head_tree), Ok(base_tree)) = (
632                git::tree_of(&worktree, "HEAD").await,
633                git::tree_of(&worktree, &base_commit).await,
634            )
635            && head_tree == base_tree
636        {
637            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
638            git::worktree_remove(repo, &worktree).await.ok();
639            bail!(
640                "`{branch}` at {} has a tree identical to base {}; this usually means \
641                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
642                 against `{}/{branch}`) rather than an empty change",
643                short(&head),
644                short(&base_commit),
645                state.config.merge.remote
646            );
647        }
648        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
649            .await
650            .unwrap_or_default();
651
652        state.candidates.push(Candidate {
653            index: 0,
654            label: 'A',
655            // Not an agent id on purpose: nothing in the roster wrote this, and
656            // the stats tables must not credit anyone with a win for it.
657            agent: "(existing branch)".to_owned(),
658            branch: branch.to_owned(),
659            worktree,
660            summary: String::new(),
661            stat,
662            files,
663            commits,
664            empty: false,
665            failed: None,
666            verified_noop: None,
667            duration_ms: 0,
668            folded: false,
669        });
670        state.tally = Some(Tally {
671            first_choice: BTreeMap::from([('A', 0)]),
672            borda: BTreeMap::new(),
673            winner: 'A',
674            rankings: 0,
675            unanimous_initial: false,
676            deliberated: false,
677            changed_votes: 0,
678            unanimous_final: false,
679            tie_break: None,
680            // No panel sat, so no quorum applies. Zero judges is the correct
681            // number for work that never competed, and must not be reported as
682            // a collapsed panel.
683            judges: 0,
684            present: 0,
685            quorum: 0,
686            met_quorum: true,
687            uncontested: Some("review-only run: nothing competed".to_owned()),
688        });
689        state.status = RunStatus::Reviewing;
690        state.event(
691            "start",
692            format!(
693                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
694                state.id
695            ),
696        );
697        state.save()?;
698        Ok(Self {
699            state,
700            roles,
701            sem: Arc::new(Semaphore::new(max_parallel)),
702            pause: Pause::new(),
703            interrupt: Pause::new(),
704        })
705    }
706
707    /// Reopen an existing run.
708    pub fn resume(id: &str) -> Result<Self> {
709        let state = RunState::load(id)?;
710        if let Some(to) = &state.released_to {
711            bail!(
712                "run {} cannot be resumed: its worktree was released to run {}",
713                state.short(),
714                crate::run::short_of(to)
715            );
716        }
717        let roles = state.config.resolve_roles()?;
718        let max_parallel = state.config.graph.max_parallel.max(1);
719        Ok(Self {
720            state,
721            roles,
722            sem: Arc::new(Semaphore::new(max_parallel)),
723            pause: Pause::new(),
724            interrupt: Pause::new(),
725        })
726    }
727
728    /// Walk the graph to a terminal state, skipping nodes already recorded.
729    ///
730    /// Every way a run is driven - the queue loop, `magi run`, a resume from
731    /// the phone - ends here, so this is the one place a run that ended
732    /// Blocked / Stalled / Failed, or died with an error, is announced to the
733    /// notification centre. Best-effort: see [`crate::notices::raise`].
734    pub async fn execute(&mut self) -> Result<()> {
735        let result = self.execute_graph().await;
736        self.mark_driver_exited();
737        let ended = if result.is_err() {
738            Some(crate::notices::run_stopped(&self.state.id, &self.state))
739        } else {
740            crate::notices::run_ended(&self.state)
741        };
742        if let Some(notice) = ended {
743            crate::notices::raise(notice);
744        }
745        result
746    }
747
748    /// Record that this process no longer drives the run, so its pid (a
749    /// daemon's outlives the run) is not read as a live driver.
750    ///
751    /// Written onto the record as it is on disk, never this copy: another
752    /// process may have resumed the run (recording its own pid and clearing
753    /// the flag) or released its worktree since this copy was read, and
754    /// saving over that would mark a running driver dead. Only a record still
755    /// naming this process as the driver is touched.
756    fn mark_driver_exited(&mut self) {
757        self.state.driver_exited = true;
758        let pid = std::process::id();
759        let Ok(mut disk) = RunState::load(&self.state.id) else {
760            return;
761        };
762        if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
763            return;
764        }
765        disk.driver_exited = true;
766        if let Err(e) = disk.save() {
767            tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
768        }
769    }
770
771    async fn execute_graph(&mut self) -> Result<()> {
772        // Moving again, so it is no longer parked. Set before the walk rather
773        // than in `resume`, so every way of re-entering the graph clears it
774        // and a card cannot claim a run is waiting to be resumed while the
775        // agents are already working.
776        self.state.parked = false;
777        // Any seat this state still lists as answering belongs to whatever
778        // process last drove this run — this one included, if it crashed
779        // mid-wave. Cleared and flushed immediately, before anything else
780        // runs, so a resume can never show a seat as live when nothing is
781        // asking it anything yet; the node that actually dispatches the next
782        // wave repopulates it.
783        self.state.clear_active();
784        // Recorded in the same spot, and flushed together with the clear
785        // above: this is the pid a reader checks (`RunState::liveness`) when
786        // no daemon claim exists to answer "is a process still driving this
787        // run" — a plain `magi run` / `magi review` typed into a terminal
788        // claims nothing there. Always overwritten, never only-if-absent, so
789        // a resumed run's stale pid from a previous, possibly-dead process
790        // can never survive into this one's own report. Unlike
791        // `clear_active`, this changes on every single `execute()` call, so
792        // the save below is now unconditional rather than only-if-cleared.
793        //
794        // `driver_started_at` is recorded in the same breath, from this same
795        // pid, so `liveness` can tell a live pid that is genuinely still us
796        // apart from one the OS has since handed to an unrelated process —
797        // see that field's own doc for why the pid alone is not enough.
798        // A resume that raced a takeover: the record on disk says the worktree
799        // was handed to a later run after this copy was read. Saving over it
800        // would erase that and drive a run with nothing to run in.
801        if let Ok(disk) = RunState::load(&self.state.id)
802            && let Some(to) = &disk.released_to
803        {
804            bail!(
805                "run {} cannot continue: its worktree was released to run {}",
806                self.state.short(),
807                crate::run::short_of(to)
808            );
809        }
810        let pid = std::process::id();
811        self.state.driver_pid = Some(pid);
812        self.state.driver_started_at = crate::proc::process_started_at(pid);
813        self.state.driver_exited = false;
814        self.state.save()?;
815        // A run that already lost its quorum never resumes into the verdict
816        // machinery: `deliberate` and `vote` would otherwise clobber the
817        // stalled marker back to Voting and the run would keep going past a
818        // verdict that is no longer trustworthy. Everything already recorded is
819        // kept, so the run stays resumable (or foldable) for a human to pick up.
820        //
821        // On --resume the run gets one chance to repair itself: the seats a
822        // rate limit took out are re-asked. If their quota has since reset and
823        // the quorum is restored, the run picks up and finishes; otherwise it
824        // stays stale and still-resumable for a later retry. If it does not
825        // recover, the returned status stays `Stalled` and nothing was
826        // clobbered (the recovery only mutates entries for the lost seats).
827        if self.state.status == RunStatus::Stalled {
828            if self.recover_stall().await? {
829                self.finish_after_tally().await?;
830            } else {
831                // Still below quorum: persist the marker and stay resumable.
832                self.state.save()?;
833            }
834            return Ok(());
835        }
836        // A run parked inside `land` - watching CI, mid fix-round, or
837        // waiting on the owner's merge approval - resumes directly into it,
838        // never back through `prep`. Everything before `merge` already
839        // concluded; that is the only way `status` reaches `Landing` in the
840        // first place. Re-walking `review_loop` first would also be actively
841        // wrong: its own status recomputation (see its doc) treats any
842        // clean round as reason to set `status` to `Gating`, which would
843        // clobber this marker before `merge` ever ran, and this run would
844        // never find its way back into `land` at all.
845        if self.state.status == RunStatus::Landing {
846            self.run_land().await?;
847            // `run_land` may have settled the run right here - CI came back
848            // green and the PR merged, say - without ever passing back
849            // through `merge`'s own trailing call. Whatever it left `status`
850            // as is what this has to read.
851            self.settle_questions();
852            return Ok(());
853        }
854        self.prep().await?;
855        if self.park_here()? {
856            return Ok(());
857        }
858        self.advise().await?;
859        if self.park_here()? {
860            return Ok(());
861        }
862        self.implement().await?;
863        if self.park_here()? {
864            return Ok(());
865        }
866        // `after_implement` already saved the state and settled any open
867        // questions when it set this; nothing later in the graph has
868        // anything to judge.
869        if self.state.status == RunStatus::VerifiedNoop {
870            return Ok(());
871        }
872        self.judge().await?;
873        if self.park_here()? {
874            return Ok(());
875        }
876        self.deliberate().await?;
877        if self.park_here()? {
878            return Ok(());
879        }
880        self.vote().await?;
881        if self.park_here()? {
882            return Ok(());
883        }
884        self.tally()?;
885        // A verdict that lost its quorum is not trustworthy: do not review,
886        // gate, or merge on it. Everything already done is kept, so the run
887        // stays resumable (or foldable); the human can replace the agent that
888        // ran out of quota and pick it up.
889        if self.state.status == RunStatus::Stalled {
890            // Persist the stalled marker now — the normal end-of-execute save
891            // below is below this early return, and without it a resumed run
892            // would reload a pre-tally status and keep going.
893            self.state.save()?;
894            return Ok(());
895        }
896        self.finish_after_tally().await?;
897        Ok(())
898    }
899
900    /// Park here if asked to, recording it in the run's own timeline.
901    ///
902    /// Returns whether the caller should stop walking the graph. The state is
903    /// saved either way by the node that just finished; this adds the event so
904    /// the operator's card says why a run that is neither finished nor moving
905    /// is sitting where it is.
906    fn park_here(&mut self) -> Result<bool> {
907        // Either handle asking is enough - see `Pause`'s own doc for why
908        // they are never the same one. `interrupt` is checked second so a
909        // reason it carries is preferred in the message below over a plain
910        // shutdown park racing it at the same boundary.
911        if !self.pause.parked() && !self.interrupt.parked() {
912            return Ok(false);
913        }
914        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
915            Some(reason) => format!(
916                "parked after `{}` ({reason}) — resume to carry on from here",
917                self.state.status.as_str()
918            ),
919            None => format!(
920                "parked after `{}` — resume to carry on from here",
921                self.state.status.as_str()
922            ),
923        };
924        self.state.event("park", why);
925        self.state.parked = true;
926        self.state.save()?;
927        Ok(true)
928    }
929
930    /// Hand the runner the pause `magi serve`'s own shutdown watches.
931    pub fn on_pause(&mut self, pause: Pause) {
932        self.pause = pause;
933    }
934
935    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
936    /// scheduler asking this one run - and no other - to park so a task
937    /// marked [`crate::queue::Task::interrupt`] can run alone. See
938    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
939    /// handle.
940    pub fn watch_interrupt(&mut self, pause: Pause) {
941        self.interrupt = pause;
942    }
943
944    /// Abandon this run's own open questions, once `status` has actually
945    /// settled rather than merely paused.
946    ///
947    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
948    /// either back up with the candidates, the review round and the seat
949    /// sessions already on disk, so a question an implementer asked mid-round
950    /// may still get a real answer read by a real resume. Only the statuses
951    /// `resumable` excludes are actually final: the run merged, it reached
952    /// `Ready` with nothing left to do, it failed outright with no
953    /// established point to continue from, or every candidate agreed, with
954    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
955    /// every one of those the seat that asked is gone for good, exactly like
956    /// the run being deleted under `magi run rm` - so the same cleanup
957    /// applies, worded for what actually happened instead of "the run was
958    /// deleted".
959    ///
960    /// Best-effort and silent on success: called from every place `status`
961    /// can land on one of those three, including ones a resumed run revisits,
962    /// so it must cost nothing when there was nothing open to begin with.
963    fn settle_questions(&mut self) {
964        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
965            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
966        }
967    }
968
969    /// The tail of the graph after a trustworthy tally: fold losers, review,
970    /// gate, merge, and persist.
971    async fn finish_after_tally(&mut self) -> Result<()> {
972        self.fold_losers().await?;
973        // Before review starts, and again right before the gate: a run's
974        // review rounds can themselves take long enough for the base to move
975        // a second time, and the gate is the one node whose "green" gets
976        // acted on.
977        self.sync_to_base().await?;
978        self.review_loop().await?;
979        self.sync_to_base().await?;
980        self.gate().await?;
981        self.merge().await?;
982        self.state.save()?;
983        Ok(())
984    }
985
986    // ---------------------------------------------------------------- prep
987
988    async fn prep(&mut self) -> Result<()> {
989        if !self.state.candidates.is_empty() {
990            return Ok(());
991        }
992        self.state.status = RunStatus::Prep;
993        let repo = self.state.repo.clone();
994        let base = self.state.base_commit.clone();
995        let root = self.state.worktree_root();
996        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
997
998        // The hook is the write-time half of the blindness contract; the
999        // presentation filter in `blind` is the half that cannot be bypassed.
1000        let hooks_dir = self.state.dir().join("hooks");
1001        if self.state.config.blind.commit_msg_hook {
1002            std::fs::create_dir_all(&hooks_dir)
1003                .with_context(|| format!("create {}", hooks_dir.display()))?;
1004            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1005            let path = hooks_dir.join("commit-msg");
1006            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1007            make_executable(&path)?;
1008            // Ref-counted rather than a plain idempotent set: with more than
1009            // one run able to be in flight in the same repository at once
1010            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1011            // true?" check cannot tell "another run of mine still needs
1012            // this" from "nobody does", and the run that happens to finish
1013            // first would disable the hook out from under a sibling still
1014            // relying on it.
1015            git::acquire_worktree_config(&repo).await?;
1016            self.state.enabled_worktree_config = true;
1017        }
1018
1019        for (index, (spec, label)) in self
1020            .roles
1021            .implementers
1022            .clone()
1023            .into_iter()
1024            .zip(labels)
1025            .enumerate()
1026        {
1027            let branch = self.state.branch_for(label);
1028            let worktree = root.join(format!("cand-{label}"));
1029            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
1030            if self.state.config.blind.commit_msg_hook {
1031                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1032            }
1033            git::local_exclude(&worktree, "/.magi/").await?;
1034            self.state.candidates.push(Candidate {
1035                index,
1036                label,
1037                agent: spec.id.clone(),
1038                branch,
1039                worktree,
1040                summary: String::new(),
1041                stat: String::new(),
1042                files: 0,
1043                commits: 0,
1044                empty: false,
1045                failed: None,
1046                verified_noop: None,
1047                duration_ms: 0,
1048                folded: false,
1049            });
1050        }
1051
1052        for j in 1..=self.roles.judges.len() {
1053            let wt = root.join(format!("judge-{j}"));
1054            if !wt.exists() {
1055                git::worktree_add_detached(&repo, &wt, &base).await?;
1056            }
1057        }
1058
1059        // Disposable, detached checkouts for the design-deliberation stage's
1060        // advisor seats — the same shape as the judges' above, at the same
1061        // base commit, since advisors also only ever read. Sized off the
1062        // configured count directly rather than a resolved roster: unlike
1063        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1064        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1065        // `prep` has no `ResolvedRoles` field to read a count from here.
1066        if self.state.config.graph.advise {
1067            for k in 1..=self.state.config.graph.advisors {
1068                let wt = root.join(format!("advisor-{k}"));
1069                if !wt.exists() {
1070                    git::worktree_add_detached(&repo, &wt, &base).await?;
1071                }
1072            }
1073        }
1074
1075        // A judge cannot tell it is looking at its own patch — the seats keep
1076        // separate conversations — but a panel that shares agents with the
1077        // field is less independent than it looks, and that is worth saying out
1078        // loud once per run rather than leaving it in the config.
1079        let authors: Vec<&str> = self
1080            .roles
1081            .implementers
1082            .iter()
1083            .map(|a| a.id.as_str())
1084            .collect();
1085        let overlap: Vec<String> = self
1086            .roles
1087            .judges
1088            .iter()
1089            .enumerate()
1090            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1091            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1092            .collect();
1093        if !overlap.is_empty() {
1094            let note = format!(
1095                "{} also authored a candidate; blind, but the panel is less \
1096                 independent than {} distinct agents would be",
1097                overlap.join(", "),
1098                self.roles.judges.len()
1099            );
1100            self.state.event("prep", note);
1101        }
1102
1103        self.state.event(
1104            "prep",
1105            format!(
1106                "{} candidates, {} judges, base {} ({})",
1107                self.state.candidates.len(),
1108                self.roles.judges.len(),
1109                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1110                self.state.base_branch
1111            ),
1112        );
1113        self.state.status = RunStatus::Implementing;
1114        self.state.save()?;
1115        Ok(())
1116    }
1117
1118    // -------------------------------------------------------------- advise
1119
1120    /// The design-deliberation stage: independent, read-only advisor seats
1121    /// each sketch a design before any implementer touches the repository,
1122    /// and (when at least one produced a usable proposal) a synthesis seat
1123    /// blends them into a brief `implement` carries in every candidate's
1124    /// prompt.
1125    ///
1126    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1127    /// advisors` is the proposal count. Everything here is best-effort and
1128    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1129    /// that cannot reach quota, or a synthesis seat that produced nothing
1130    /// usable all leave `implement` exactly as it was before this stage
1131    /// existed — the task instruction alone — rather than failing the whole
1132    /// competition over an enrichment stage. Every outcome is still recorded
1133    /// as an event, so a run that got nothing from this stage says why.
1134    ///
1135    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1136    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1137    /// resumed run whose stage failed would re-run it, and re-spend the
1138    /// agent calls, on every reentry before `implement`.
1139    ///
1140    /// Also skipped once any candidate shows implementation progress — the
1141    /// exact predicate `implement` itself uses to decide a candidate is no
1142    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1143    /// is not enough: a run created by an older binary that predates this
1144    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1145    /// an already-`Implementing`-or-later run under this build would
1146    /// otherwise walk straight back through `prep` (a no-op once candidates
1147    /// exist) into this node and spawn every advisor seat against worktrees
1148    /// `prep` never recreated — after implementation has already started,
1149    /// which is exactly the invariant this stage exists to guarantee.
1150    async fn advise(&mut self) -> Result<()> {
1151        let implement_untouched = self
1152            .state
1153            .candidates
1154            .iter()
1155            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1156        if !self.state.config.graph.advise || self.state.advise_attempted {
1157            return Ok(());
1158        }
1159        if !implement_untouched {
1160            self.state.event(
1161                "advise",
1162                "skipping the design-deliberation stage: at least one \
1163                 candidate already shows implementation progress, so this \
1164                 run is past the point the stage exists to run before"
1165                    .to_owned(),
1166            );
1167            self.state.advise_attempted = true;
1168            self.state.save()?;
1169            return Ok(());
1170        }
1171        let run_id = self.state.id.clone();
1172        let prompts = self.state.config.prompts.clone();
1173        let instruction = self.state.instruction.clone();
1174        let language = self.state.config.graph.language.clone();
1175        let root = self.state.worktree_root();
1176        let n = self.state.config.graph.advisors;
1177        let where_recorded = self.state.dir().join("run.json");
1178
1179        let seats = match self.state.config.advisors() {
1180            Ok(seats) if !seats.is_empty() => seats,
1181            Ok(_) => {
1182                self.state.event(
1183                    "advise",
1184                    format!(
1185                        "[graph] advisors is 0; skipping the design-deliberation \
1186                         stage and continuing without a synthesis brief (see {})",
1187                        where_recorded.display()
1188                    ),
1189                );
1190                self.state.advise_attempted = true;
1191                self.state.save()?;
1192                return Ok(());
1193            }
1194            Err(e) => {
1195                self.state.event(
1196                    "advise",
1197                    format!(
1198                        "could not resolve advisor seats ({e:#}); continuing \
1199                         without a design-deliberation brief (see {})",
1200                        where_recorded.display()
1201                    ),
1202                );
1203                self.state.advise_attempted = true;
1204                self.state.save()?;
1205                return Ok(());
1206            }
1207        };
1208
1209        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1210        let artifacts = agent::artifacts_dir(&self.state.dir());
1211        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1212
1213        let mut jobs = Vec::new();
1214        for (i, spec) in seats.iter().cloned().enumerate() {
1215            let seat_key = format!("advisor-{}", i + 1);
1216            let seat = self.seat(&seat_key, &spec.id);
1217            jobs.push(SeatJob {
1218                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1219                spec,
1220                seat,
1221                cwd: worktrees[i % worktrees.len()].clone(),
1222                timeout,
1223                allow_write: false,
1224                sessions: false,
1225                artifacts: artifacts.clone(),
1226                stem: seat_key,
1227            });
1228        }
1229
1230        self.state.event(
1231            "advise",
1232            format!(
1233                "{} advisor seat(s) sketching a design in parallel",
1234                jobs.len()
1235            ),
1236        );
1237        let mut quota_losses = Vec::new();
1238        let cache = self.state.config.cache_dir();
1239        let ctx = WaveCtx {
1240            run: &run_id,
1241            node: "advise",
1242            prompts: &prompts,
1243            cache: cache.as_deref(),
1244            round: None,
1245        };
1246        let results = ask_json_wave::<Proposal>(
1247            jobs,
1248            Arc::clone(&self.sem),
1249            self.state.config.graph.retries,
1250            &ctx,
1251            &mut quota_losses,
1252            &mut self.state,
1253            &|p: &Proposal| p.validate(),
1254        )
1255        .await;
1256        self.state.quota.extend(quota_losses);
1257
1258        let mut records = Vec::with_capacity(results.len());
1259        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1260            let agent_id = seat.agent.clone();
1261            self.state.seats.insert(seat.key.clone(), seat);
1262            match res {
1263                Ok((proposal, out)) => {
1264                    self.state
1265                        .event("advise", format!("advisor-{} proposed a design", i + 1));
1266                    records.push(advise::AdvisorRecord::proposed(
1267                        i + 1,
1268                        agent_id,
1269                        proposal,
1270                        out.duration_ms,
1271                    ));
1272                }
1273                Err(e) => {
1274                    self.state.event(
1275                        "advise",
1276                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1277                    );
1278                    records.push(advise::AdvisorRecord::failed(
1279                        i + 1,
1280                        agent_id,
1281                        e.to_string(),
1282                    ));
1283                }
1284            }
1285        }
1286
1287        let mut advice = advise::Advice {
1288            records,
1289            synthesis: None,
1290        };
1291        if advice.proposals().is_empty() {
1292            self.state.event(
1293                "advise",
1294                "no advisor produced a usable proposal; continuing without a \
1295                 synthesis brief"
1296                    .to_owned(),
1297            );
1298        } else {
1299            match self
1300                .synthesize_brief(
1301                    &advice,
1302                    &instruction,
1303                    &language,
1304                    &worktrees[0],
1305                    &artifacts,
1306                    &run_id,
1307                    &prompts,
1308                    cache.as_deref(),
1309                )
1310                .await
1311            {
1312                Ok(Some(text)) => {
1313                    self.state.event(
1314                        "advise",
1315                        "synthesized a design brief for the implementer".to_owned(),
1316                    );
1317                    advice.synthesis = Some(text);
1318                }
1319                Ok(None) => {
1320                    self.state.event(
1321                        "advise",
1322                        "the synthesis seat produced nothing usable; continuing \
1323                         without a design brief"
1324                            .to_owned(),
1325                    );
1326                }
1327                Err(e) => {
1328                    self.state.event(
1329                        "advise",
1330                        format!("could not synthesize a design brief: {e:#}"),
1331                    );
1332                }
1333            }
1334        }
1335        advise::apply_reflection(&mut advice);
1336
1337        self.state.advice = Some(advice);
1338        self.state.advise_attempted = true;
1339        self.state.save()?;
1340        Ok(())
1341    }
1342
1343    /// The synthesis seat: reads every advisor's proposal and blends them
1344    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1345    /// out of [`Runner::advise`] only for readability — it is not called
1346    /// anywhere else.
1347    ///
1348    /// Picked the same way [`crate::talk`]'s standing conversation and
1349    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1350    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1351    /// order (a claude seat, else the first runnable agent in roster order)
1352    /// used when that field is unset — see `[roles] synthesizer`'s own doc
1353    /// in [`crate::config`] for why a dedicated field exists here at all.
1354    #[allow(clippy::too_many_arguments)]
1355    async fn synthesize_brief(
1356        &mut self,
1357        advice: &advise::Advice,
1358        instruction: &str,
1359        language: &str,
1360        cwd: &Path,
1361        artifacts: &Path,
1362        run_id: &str,
1363        prompts: &Prompts,
1364        cache: Option<&Path>,
1365    ) -> Result<Option<String>> {
1366        let want = self.state.config.roles.synthesizer.as_deref();
1367        let spec = agent::pick(&self.state.config.agents, want, &agent::installed)?;
1368        let mut seat = self.seat("advise-synthesis", &spec.id);
1369        let proposals = advice.proposals();
1370        let mut prompt = prompt::with_overlay(
1371            prompt::synthesize_brief(instruction, &proposals, language),
1372            prompts.overlay("advise"),
1373        );
1374        if cache.is_some() {
1375            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1376            // below — see `prompt::build_cache_note`'s doc for why telling a
1377            // read-only seat to build through the shared cache is exactly how
1378            // a sandbox's write refusal gets misread as a defect.
1379            prompt.push('\n');
1380            prompt.push_str(&prompt::build_cache_note("advise", false));
1381        }
1382        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1383        let out = agent::invoke(
1384            &spec,
1385            &mut seat,
1386            &Invocation {
1387                cwd,
1388                prompt: &prompt,
1389                timeout,
1390                allow_write: false,
1391                sessions: false,
1392                artifacts,
1393                stem: "advise-synthesis",
1394                run: run_id,
1395                node: "advise",
1396                cache_dir: None,
1397                attachments: &[],
1398            },
1399        )
1400        .await?;
1401        self.state.seats.insert(seat.key.clone(), seat);
1402        if !out.usable() {
1403            return Ok(None);
1404        }
1405        let text =
1406            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1407        Ok((!text.trim().is_empty()).then_some(text))
1408    }
1409
1410    // ----------------------------------------------------------- implement
1411
1412    async fn implement(&mut self) -> Result<()> {
1413        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1414        // agent files with `magi task add` name the run that paid for it. The
1415        // prompt overlay is cloned alongside it because the waves borrow it
1416        // while `self` is mutably borrowed by the node's own bookkeeping.
1417        let run_id = self.state.id.clone();
1418        let prompts = self.state.config.prompts.clone();
1419        let todo: Vec<usize> = self
1420            .state
1421            .candidates
1422            .iter()
1423            .enumerate()
1424            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1425            .map(|(i, _)| i)
1426            .collect();
1427        if todo.is_empty() {
1428            return self.after_implement();
1429        }
1430        self.state.status = RunStatus::Implementing;
1431
1432        let language = self.state.config.graph.language.clone();
1433        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1434        let sessions = self.state.config.graph.sessions;
1435        let artifacts = agent::artifacts_dir(&self.state.dir());
1436        // The design-deliberation stage's blended brief, when `advise` found
1437        // one — carried into every implementer's prompt the same way
1438        // regardless of which candidate it is.
1439        let brief = self
1440            .state
1441            .advice
1442            .as_ref()
1443            .and_then(|a| a.synthesis.as_deref())
1444            .map(str::to_owned);
1445
1446        let mut jobs = Vec::new();
1447        for &i in &todo {
1448            let (index, label, worktree) = {
1449                let c = &self.state.candidates[i];
1450                (c.index, c.label, c.worktree.clone())
1451            };
1452            let spec = self.roles.implementers[index].clone();
1453            let seat_key = format!("impl-{label}");
1454            let seat = self.seat(&seat_key, &spec.id);
1455            let instruction = self.state.instruction.clone();
1456            jobs.push(SeatJob {
1457                spec,
1458                seat,
1459                prompt: prompt::implement(
1460                    &instruction,
1461                    &worktree.to_string_lossy(),
1462                    &language,
1463                    brief.as_deref(),
1464                ),
1465                cwd: worktree,
1466                timeout,
1467                allow_write: true,
1468                sessions,
1469                artifacts: artifacts.clone(),
1470                stem: format!("impl-{label}"),
1471            });
1472        }
1473
1474        self.state.event(
1475            "implement",
1476            format!("{} candidates in parallel", jobs.len()),
1477        );
1478        // Kept so a seat whose CLI hung up can be asked again from the same
1479        // job: `wave` consumes what it is given. Mutable so `resume_quota_losses`
1480        // can update a seat's own entry once a fallback agent takes it over —
1481        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1482        // whichever agent actually answered, not the one that quota'd out.
1483        let mut sent = jobs.clone();
1484        let cache = self.state.config.cache_dir();
1485        let ctx = WaveCtx {
1486            run: &run_id,
1487            node: "implement",
1488            prompts: &prompts,
1489            cache: cache.as_deref(),
1490            round: None,
1491        };
1492        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1493        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1494            .await;
1495        self.resume_quota_losses(&mut results, &mut sent, &prompts, &run_id)
1496            .await;
1497        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1498            .await;
1499
1500        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1501            let seat_key = seat.key.clone();
1502            // A quota fallback (`resume_quota_losses`) may have handed this
1503            // seat to a different agent than the one `prep` recorded on the
1504            // candidate; the stats tables and any later fixer-defaults-to-
1505            // winner's-author lookup must credit whoever actually answered —
1506            // unless every fallback also quota'd out, in which case nobody
1507            // actually answered and crediting the last agent tried would
1508            // erase every earlier agent's own quota loss from the stats
1509            // tables instead of just this one seat's.
1510            let agent = seat.agent.clone();
1511            let exhausted_the_fallback_chain = matches!(&out, AgentOutcome::Quota(_));
1512            self.state.seats.insert(seat.key.clone(), seat);
1513            let label = self.state.candidates[i].label;
1514            let worktree = self.state.candidates[i].worktree.clone();
1515            let base = self.state.base_commit.clone();
1516
1517            let (summary, duration, failed, verified_claim) = match out {
1518                AgentOutcome::Ok(o) => {
1519                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1520                    let failed = (!o.usable()).then(|| {
1521                        if o.timed_out {
1522                            "agent timed out".to_owned()
1523                        } else {
1524                            format!("agent exited with {:?}", o.exit_code)
1525                        }
1526                    });
1527                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1528                    (text, o.duration_ms, failed, verified_claim)
1529                }
1530                // Left un-resumed by `resume_undelivered` (a dirty tree
1531                // already rescues the work, or there was no session left to
1532                // resume into) — reported like the ordinary failure it is,
1533                // never as if `o.text` (the CLI's raw error JSON) were an
1534                // answer.
1535                AgentOutcome::Dropped(o) => {
1536                    let why = o
1537                        .dropped
1538                        .as_ref()
1539                        .map(|d| d.why.as_str())
1540                        .unwrap_or("the CLI ended the stream without delivering its answer");
1541                    (
1542                        String::new(),
1543                        o.duration_ms,
1544                        Some(format!("the CLI dropped the stream ({why})")),
1545                        None,
1546                    )
1547                }
1548                AgentOutcome::Quota(o) => {
1549                    self.state.quota.push(QuotaLoss {
1550                        seat: seat_key,
1551                        node: "implement".to_owned(),
1552                        at: Timestamp::now(),
1553                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1554                    });
1555                    (
1556                        String::new(),
1557                        o.duration_ms,
1558                        Some("rate limited (quota); produced no change".to_owned()),
1559                        None,
1560                    )
1561                }
1562                AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1563            };
1564
1565            // Rescue anything the agent edited but never committed: an
1566            // uncommitted candidate would silently be an empty one.
1567            let rescued = match git::rescue_commit(
1568                &worktree,
1569                &format!("magi: candidate {label} (uncommitted work)"),
1570            )
1571            .await
1572            {
1573                Ok(r) => {
1574                    self.state.note_withheld("implement", &r.withheld);
1575                    r.committed
1576                }
1577                Err(_) => false,
1578            };
1579            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1580                .await
1581                .unwrap_or(0);
1582            let patch = git::diff(&worktree, &base, "HEAD")
1583                .await
1584                .unwrap_or_default();
1585            let stat = git::diff_stat(&worktree, &base, "HEAD")
1586                .await
1587                .unwrap_or_default();
1588            let files = git::changed_files(&worktree, &base, "HEAD")
1589                .await
1590                .map(|f| f.len())
1591                .unwrap_or(0);
1592            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1593
1594            let c = &mut self.state.candidates[i];
1595            if !exhausted_the_fallback_chain {
1596                c.agent = agent;
1597            }
1598            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1599            c.stat = stat;
1600            c.files = files;
1601            c.commits = commits;
1602            c.duration_ms = duration;
1603            c.empty = commits == 0 || patch.trim().is_empty();
1604            // An agent that failed but still produced a committed change stays
1605            // in the running: the patch is what gets judged, not the exit code.
1606            c.failed = match failed {
1607                Some(_) if c.empty => failed,
1608                _ => None,
1609            };
1610            // Only an empty candidate can be a verified no-op: a claim next
1611            // to a real patch is not what the marker is for, and `c.failed`
1612            // being `Some` here already implies `verified_claim` was never
1613            // set (see the guard above the match that produced it).
1614            c.verified_noop = if c.empty { verified_claim } else { None };
1615            let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1616                (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1617                (None, true, Some(_), _) => {
1618                    format!("candidate {label}: no change produced (agent-verified no-op)")
1619                }
1620                (None, true, None, _) => format!("candidate {label}: no change produced"),
1621                (None, false, _, true) => {
1622                    format!(
1623                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1624                    )
1625                }
1626                (None, false, _, false) => {
1627                    format!("candidate {label}: {files} files, {commits} commits")
1628                }
1629            };
1630            self.state.event("implement", note);
1631            self.state.save()?;
1632        }
1633
1634        self.after_implement()
1635    }
1636
1637    /// Ask again, once, for work a CLI did and then failed to hand over.
1638    ///
1639    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1640    /// status with an empty response and a usage report showing output tokens,
1641    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1642    /// seven minutes and 14,267 output tokens that arrived as an empty
1643    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1644    ///
1645    /// Two conditions, and both matter:
1646    ///
1647    /// - **Only when the tree is untouched.** Often the agent has already
1648    ///   written its files and only the closing message was lost; the rescue
1649    ///   commit below picks that up and there is nothing to ask for. Re-asking
1650    ///   then would pay for a second implementation of work already on disk.
1651    /// - **Once.** A CLI that drops one stream can drop the next, and this
1652    ///   node is the most expensive in the graph.
1653    ///
1654    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1655    /// dropped reply still carried its `conversation_id`, so the seat is asked
1656    /// to finish what it was doing rather than sent the whole task again. It
1657    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1658    /// node's - for the same reason a re-ranked judge does: restating finished
1659    /// work is not the work.
1660    ///
1661    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1662    /// same way until it resets, while an abandoned conversation is still
1663    /// there to be picked up.
1664    async fn resume_undelivered(
1665        &mut self,
1666        results: &mut [(usize, SeatState, AgentOutcome)],
1667        sent: &[SeatJob],
1668        prompts: &Prompts,
1669        run_id: &str,
1670    ) {
1671        for (wi, seat, out) in results.iter_mut() {
1672            let Some(dropped) = (match &*out {
1673                AgentOutcome::Dropped(o) => o.dropped.clone(),
1674                _ => None,
1675            }) else {
1676                continue;
1677            };
1678            let Some(job) = sent.get(*wi) else { continue };
1679            // Already on disk? Then only the closing message was lost.
1680            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1681                self.state.event(
1682                    "implement",
1683                    format!(
1684                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1685                         work is in the tree",
1686                        seat.key, dropped.output_tokens, dropped.why
1687                    ),
1688                );
1689                continue;
1690            }
1691            // The re-ask only makes sense as a resume: `resume_after_drop`
1692            // says nothing about the task, trusting the seat to still hold it.
1693            // Without a session to resume — sessions disabled, or this CLI's
1694            // drop shape happened not to carry a session id — that prompt
1695            // would open a brand-new conversation with no context at all,
1696            // which is worse than leaving this as the ordinary failure it
1697            // already is.
1698            if !has_context(&job.spec, seat, job.sessions) {
1699                self.state.event(
1700                    "implement",
1701                    format!(
1702                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1703                         is no session left to resume",
1704                        seat.key, dropped.output_tokens, dropped.why
1705                    ),
1706                );
1707                continue;
1708            }
1709            self.state.event(
1710                "implement",
1711                format!(
1712                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1713                     conversation",
1714                    seat.key, dropped.output_tokens, dropped.why
1715                ),
1716            );
1717            let mut retry = job.clone();
1718            retry.seat = seat.clone();
1719            retry.prompt = prompt::resume_after_drop(&dropped.why);
1720            retry.timeout = retry_budget(job.timeout, true);
1721            retry.stem = format!("{}-resume", job.stem);
1722            let cache = self.state.config.cache_dir();
1723            let ctx = WaveCtx {
1724                run: run_id,
1725                node: "implement",
1726                prompts,
1727                cache: cache.as_deref(),
1728                round: None,
1729            };
1730            let (resumed_seat, resumed) =
1731                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1732            *seat = resumed_seat;
1733            *out = resumed;
1734        }
1735    }
1736
1737    /// Fall an implement seat through to the next untried agent in the
1738    /// implementer roster when it lost to quota, instead of leaving the
1739    /// seat's loss final the moment one agent's account runs dry.
1740    ///
1741    /// Solo runs (`graph.candidates = 1`, `daemon::apply_solo`'s forced shape)
1742    /// are the motivating case: `Config::resolve_roles`'s `implementers`
1743    /// truncates to the single slot rotation picked, so a solo task whose one
1744    /// implementer hits quota mid-run used to have nothing else to try. This
1745    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
1746    /// unrotated roster — which is the only place the *other* candidates in
1747    /// the machine's roster still exist once `implementers` has been cut down
1748    /// to size.
1749    ///
1750    /// Walks forward from just past the seat's own original position in the
1751    /// roster, never wrapping back to the front: a later candidate slot (say
1752    /// `beta`, the roster's second entry) must fall through to the *next*
1753    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
1754    /// almost certainly a different candidate's own agent already — and once
1755    /// the roster's tail is exhausted there is nothing left to fall through
1756    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
1757    /// whole [`AgentSpec`]: a roster with the same id named twice must not
1758    /// let this retry that id forever. The loop keeps falling through until
1759    /// an attempt lands something other than `Quota` or the roster's tail
1760    /// runs out of untried ids, at which point the seat is left exactly as
1761    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
1762    /// `QuotaLoss` recorded, the candidate failed/empty.
1763    ///
1764    /// `sent` is taken mutably and updated with the fallback agent's spec:
1765    /// `resume_unconfirmed_commands`, which runs after this and also reads
1766    /// `sent`, must see whichever agent actually ended up answering the seat
1767    /// — reading the stale, original spec there would check session
1768    /// eligibility against the wrong CLI and could hand a fallback agent's
1769    /// session id to the agent that just lost the seat to quota.
1770    ///
1771    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
1772    /// — `self.seat` only reuses state when the agent id is unchanged, so
1773    /// handing it a different id already gets this for free. Reusing the old
1774    /// seat would resume a different CLI's session as if it were a
1775    /// continuation of this one.
1776    ///
1777    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
1778    /// a quota loss cuts an agent off mid-turn, so anything already in the
1779    /// tree is unfinished work, not a completed candidate a re-ask would pay
1780    /// for twice. A dirty tree is rescued into a commit first (the same
1781    /// neutral-identity rescue `implement`'s own outcome loop gives every
1782    /// candidate) so the next agent starts clean.
1783    ///
1784    /// The new agent gets the implementer's full prompt and full
1785    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
1786    /// it has no session and no context, and is implementing the task from
1787    /// nothing, unlike a resumed drop which is only restating work already
1788    /// done.
1789    ///
1790    /// Every intermediate `Quota` this loop absorbs is folded into a plain
1791    /// `implement` event, never into `self.state.quota` — that is what
1792    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
1793    /// go unspent, and a seat that ultimately recovered on its second or
1794    /// third agent is not the stalled panel that check exists to catch. Only
1795    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
1796    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
1797    /// outcome loop already has — this helper never pushes to it itself.
1798    async fn resume_quota_losses(
1799        &mut self,
1800        results: &mut [(usize, SeatState, AgentOutcome)],
1801        sent: &mut [SeatJob],
1802        prompts: &Prompts,
1803        run_id: &str,
1804    ) {
1805        let instruction = self.state.instruction.clone();
1806        let language = self.state.config.graph.language.clone();
1807        let brief = self
1808            .state
1809            .advice
1810            .as_ref()
1811            .and_then(|a| a.synthesis.as_deref())
1812            .map(str::to_owned);
1813        for (wi, seat, out) in results.iter_mut() {
1814            let Some(job) = sent.get_mut(*wi) else {
1815                continue;
1816            };
1817            // Where the seat's own original agent sits in the roster — the
1818            // fallback walk starts just past here, never at the front, so a
1819            // later candidate slot's quota loss does not fall back onto an
1820            // earlier slot's own agent.
1821            let start = self
1822                .roles
1823                .implementer_roster
1824                .iter()
1825                .position(|s| s.id == job.spec.id)
1826                .unwrap_or(0);
1827            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1828            let mut fallback_attempt = 0usize;
1829            while matches!(&*out, AgentOutcome::Quota(_)) {
1830                let Some(next) =
1831                    next_untried_implementer(&self.roles.implementer_roster, start, &tried)
1832                        .cloned()
1833                else {
1834                    break;
1835                };
1836                tried.insert(next.id.clone());
1837                fallback_attempt += 1;
1838
1839                if let Ok(r) = git::rescue_commit(
1840                    &job.cwd,
1841                    &format!(
1842                        "magi: candidate {} (uncommitted work before quota fallback)",
1843                        seat.key
1844                    ),
1845                )
1846                .await
1847                {
1848                    self.state.note_withheld("implement", &r.withheld);
1849                }
1850
1851                self.state.event(
1852                    "implement",
1853                    format!(
1854                        "{}: rate limited (quota) on {}; retrying with {}",
1855                        seat.key, seat.agent, next.id
1856                    ),
1857                );
1858
1859                let new_seat = self.seat(&seat.key, &next.id);
1860                // Kept in sync on `sent` itself, not just the local retry: a
1861                // later helper (`resume_unconfirmed_commands`) reads `sent`
1862                // after this one returns and must see whichever agent is now
1863                // occupying the seat, not the one that just quota'd out —
1864                // otherwise it would judge session/continuation eligibility
1865                // by the wrong CLI and could resend a fallback's session id
1866                // to the agent that lost it the seat in the first place.
1867                job.spec = next.clone();
1868                let mut retry = job.clone();
1869                retry.seat = new_seat;
1870                retry.prompt = prompt::implement(
1871                    &instruction,
1872                    &job.cwd.to_string_lossy(),
1873                    &language,
1874                    brief.as_deref(),
1875                );
1876                retry.stem = format!("{}-quota-{}", job.stem, next.id);
1877                let cache = self.state.config.cache_dir();
1878                let ctx = WaveCtx {
1879                    run: run_id,
1880                    node: "implement",
1881                    prompts,
1882                    cache: cache.as_deref(),
1883                    round: None,
1884                };
1885                let (fallback_seat, fallback_out) = run_one(
1886                    retry,
1887                    Arc::clone(&self.sem),
1888                    &ctx,
1889                    &mut self.state,
1890                    fallback_attempt,
1891                )
1892                .await;
1893                *seat = fallback_seat;
1894                *out = fallback_out;
1895            }
1896        }
1897    }
1898
1899    /// Ask an implement seat's own CLI to confirm what it started, once, when
1900    /// its reply reported a command whose completion status it never
1901    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1902    /// that does and does not mean.
1903    ///
1904    /// The completion contract this task asks for, extended to `implement`
1905    /// with the same signal `continue_fix_report` reads for the fixer,
1906    /// rather than a keyword search over the reply or a hard requirement on
1907    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1908    /// a candidate's CLI turn ended cleanly while a test run it had started
1909    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1910    /// named in it at all is untouched by this: `commands` is empty, so
1911    /// there is nothing to be unconfirmed.
1912    ///
1913    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1914    /// this is not about recovering edits that might already be on disk, it
1915    /// is about a result the seat itself never vouched for, which resuming
1916    /// asks for regardless of what the tree already holds. Bounded to one
1917    /// attempt for the same reason `resume_undelivered` is — this is the
1918    /// most expensive node in the graph — and a seat that still cannot
1919    /// confirm on that attempt is left as whatever its (possibly still
1920    /// unconfirmed) reply says; this does not invent a new "failed" reason
1921    /// for a candidate that otherwise produced a real, committed change.
1922    async fn resume_unconfirmed_commands(
1923        &mut self,
1924        results: &mut [(usize, SeatState, AgentOutcome)],
1925        sent: &[SeatJob],
1926        prompts: &Prompts,
1927        run_id: &str,
1928    ) {
1929        for (wi, seat, out) in results.iter_mut() {
1930            let AgentOutcome::Ok(o) = &*out else {
1931                continue;
1932            };
1933            if !has_unconfirmed_command(&o.commands) {
1934                continue;
1935            }
1936            let Some(job) = sent.get(*wi) else { continue };
1937            if !has_context(&job.spec, seat, job.sessions) {
1938                self.state.event(
1939                    "implement",
1940                    format!(
1941                        "{}: the reply named a command whose own CLI never confirmed the exit \
1942                         status of, but there is no session left to resume",
1943                        seat.key
1944                    ),
1945                );
1946                continue;
1947            }
1948            self.state.event(
1949                "implement",
1950                format!(
1951                    "{}: the reply named a command whose own CLI never confirmed the exit \
1952                     status of; resuming the conversation",
1953                    seat.key
1954                ),
1955            );
1956            let mut retry = job.clone();
1957            retry.seat = seat.clone();
1958            retry.prompt = prompt::resume_incomplete(
1959                "a command in your last reply had no confirmed exit status",
1960            );
1961            retry.timeout = retry_budget(job.timeout, true);
1962            retry.stem = format!("{}-confirm", job.stem);
1963            let cache = self.state.config.cache_dir();
1964            let ctx = WaveCtx {
1965                run: run_id,
1966                node: "implement",
1967                prompts,
1968                cache: cache.as_deref(),
1969                round: None,
1970            };
1971            let (resumed_seat, resumed) =
1972                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1973            *seat = resumed_seat;
1974            *out = resumed;
1975        }
1976    }
1977
1978    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1979    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1980    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1981    /// that motivated this.
1982    ///
1983    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1984    /// own nudge loop already covers for judge/review/vote seats, and not a
1985    /// dropped stream, which [`Runner::resume_undelivered`] covers for
1986    /// implement seats: here the CLI turn genuinely finished while the node's
1987    /// own work — the fixer's account of what it did — had not. Gated purely
1988    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
1989    /// reply, never on any wording in it, so a fixer whose valid, first-try
1990    /// `FixReport` happens to mention having waited on a background test is
1991    /// never resumed — the `Ok(report)` branch at the call site returns
1992    /// before this is ever invoked.
1993    ///
1994    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
1995    /// attempt ([`retry_budget`]), nothing attempted once the session is
1996    /// gone, and a quota hit ends the loop immediately rather than retrying a
1997    /// rate limit that fails the same way again.
1998    async fn continue_fix_report(
1999        &mut self,
2000        mut seat: SeatState,
2001        parse_err: String,
2002        job: &SeatJob,
2003        prompts: &Prompts,
2004        run_id: &str,
2005        round: usize,
2006    ) -> (
2007        SeatState,
2008        Option<FixReport>,
2009        Option<String>,
2010        ContinuationRecord,
2011    ) {
2012        let mut last_err = parse_err;
2013        let mut cumulative_wait_ms = 0u64;
2014        let mut attempts = 0usize;
2015        loop {
2016            if !has_context(&job.spec, &seat, job.sessions) {
2017                self.state.event(
2018                    "fix",
2019                    format!(
2020                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
2021                         session left to resume into"
2022                    ),
2023                );
2024                let outcome = if attempts == 0 {
2025                    ContinuationOutcome::NoSession
2026                } else {
2027                    ContinuationOutcome::Exhausted
2028                };
2029                return (
2030                    seat,
2031                    None,
2032                    Some(format!("unparsable fix report: {last_err}")),
2033                    ContinuationRecord {
2034                        attempts,
2035                        cumulative_wait_ms,
2036                        outcome,
2037                    },
2038                );
2039            }
2040            if attempts >= MAX_FIX_CONTINUATIONS {
2041                self.state.event(
2042                    "fix",
2043                    format!(
2044                        "round {round}: fixer's reply still had no adoption report after \
2045                         {attempts} continuation(s) ({last_err}); giving up"
2046                    ),
2047                );
2048                return (
2049                    seat,
2050                    None,
2051                    Some(format!(
2052                        "unparsable fix report after {attempts} continuation(s): {last_err}"
2053                    )),
2054                    ContinuationRecord {
2055                        attempts,
2056                        cumulative_wait_ms,
2057                        outcome: ContinuationOutcome::Exhausted,
2058                    },
2059                );
2060            }
2061            attempts += 1;
2062            self.state.event(
2063                "fix",
2064                format!(
2065                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2066                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2067                ),
2068            );
2069            let mut retry = job.clone();
2070            retry.seat = seat.clone();
2071            retry.prompt = prompt::resume_incomplete(&last_err);
2072            retry.timeout = retry_budget(job.timeout, true);
2073            retry.stem = format!("{}-continue{attempts}", job.stem);
2074            let cache = self.state.config.cache_dir();
2075            let ctx = WaveCtx {
2076                run: run_id,
2077                node: "fix",
2078                prompts,
2079                cache: cache.as_deref(),
2080                round: Some(round),
2081            };
2082            let (resumed_seat, resumed_out) = run_one(
2083                retry,
2084                Arc::clone(&self.sem),
2085                &ctx,
2086                &mut self.state,
2087                attempts,
2088            )
2089            .await;
2090            seat = resumed_seat;
2091            match resumed_out {
2092                AgentOutcome::Ok(o) => {
2093                    cumulative_wait_ms += o.duration_ms;
2094                    match verdict::extract_json::<FixReport>(&o.text) {
2095                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
2096                            self.state.event(
2097                                "fix",
2098                                format!(
2099                                    "round {round}: fixer's adoption report recovered after \
2100                                     {attempts} continuation(s)"
2101                                ),
2102                            );
2103                            return (
2104                                seat,
2105                                Some(report),
2106                                None,
2107                                ContinuationRecord {
2108                                    attempts,
2109                                    cumulative_wait_ms,
2110                                    outcome: ContinuationOutcome::Resumed,
2111                                },
2112                            );
2113                        }
2114                        // The report parsed, but this same reply's own
2115                        // CommandEvidence — the identical record `state.jobs`
2116                        // renders — names a command whose CLI never
2117                        // confirmed an exit status. Read together, that is
2118                        // not a resolved answer: keep nudging rather than
2119                        // accept a report standing next to a command the
2120                        // seat's own CLI cannot vouch for.
2121                        Ok(_) => {
2122                            last_err = "the reply parsed, but it reported a command whose own CLI \
2123                                 never confirmed an exit status"
2124                                .to_owned();
2125                        }
2126                        Err(e) => last_err = e.to_string(),
2127                    }
2128                }
2129                AgentOutcome::Quota(o) => {
2130                    cumulative_wait_ms += o.duration_ms;
2131                    self.state.quota.push(QuotaLoss {
2132                        seat: seat.key.clone(),
2133                        node: "fix".to_owned(),
2134                        at: Timestamp::now(),
2135                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2136                    });
2137                    self.state.event(
2138                        "fix",
2139                        format!(
2140                            "round {round}: continuation rate limited (quota); not retrying now"
2141                        ),
2142                    );
2143                    return (
2144                        seat,
2145                        None,
2146                        Some("rate limited (quota) while recovering the fix report".to_owned()),
2147                        ContinuationRecord {
2148                            attempts,
2149                            cumulative_wait_ms,
2150                            outcome: ContinuationOutcome::QuotaLost,
2151                        },
2152                    );
2153                }
2154                AgentOutcome::Dropped(o) => {
2155                    cumulative_wait_ms += o.duration_ms;
2156                    let why = o
2157                        .dropped
2158                        .as_ref()
2159                        .map(|d| d.why.as_str())
2160                        .unwrap_or("the CLI ended the stream without delivering its answer");
2161                    last_err = format!("the CLI dropped the stream ({why})");
2162                }
2163                AgentOutcome::Failed(e) => last_err = e,
2164            }
2165        }
2166    }
2167
2168    fn after_implement(&mut self) -> Result<()> {
2169        // Scan every candidate patch once the set is complete.
2170        if self.state.leaks.is_empty() {
2171            let cfg = self.state.config.blind.clone();
2172            let mut leaks = Vec::new();
2173            for c in &self.state.candidates {
2174                let Some(patch) =
2175                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2176                else {
2177                    continue;
2178                };
2179                leaks.extend(blind::scan(
2180                    &format!("candidate {} patch", c.label),
2181                    &patch,
2182                    &cfg.vendor_tokens,
2183                ));
2184            }
2185            if !leaks.is_empty() {
2186                let summary = leaks
2187                    .iter()
2188                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2189                    .collect::<Vec<_>>()
2190                    .join(", ");
2191                match cfg.on_leak {
2192                    LeakPolicy::Fail => {
2193                        self.state.status = RunStatus::Failed;
2194                        self.state
2195                            .event("blind", format!("vendor text in a patch: {summary}"));
2196                        self.state.leaks = leaks;
2197                        self.state.save()?;
2198                        self.settle_questions();
2199                        bail!(
2200                            "blind.on_leak = \"fail\" and vendor text reached a \
2201                             judged patch: {summary}"
2202                        );
2203                    }
2204                    LeakPolicy::Redact => self.state.event(
2205                        "blind",
2206                        format!("redacting vendor text for judging: {summary}"),
2207                    ),
2208                    LeakPolicy::Warn => self.state.event(
2209                        "blind",
2210                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
2211                    ),
2212                }
2213                self.state.leaks = leaks;
2214            }
2215        }
2216
2217        if self.state.viable().is_empty() {
2218            if self.state.all_candidates_verified_noop() {
2219                // Every candidate agreed, with evidence the adoption guard
2220                // accepted, that nothing belongs in this worktree. That is
2221                // not the same fact as a candidate that simply failed to
2222                // write anything, and settling it as an ordinary `Failed`
2223                // (see `SCHEMA`'s doc for schema 10) is what let two of
2224                // task 391f's attempts burn a retry each re-discovering the
2225                // same already-landed fix. Terminal either way, so `judge`
2226                // must never run over an empty candidate set — unlike the
2227                // `Failed` branch below this returns `Ok`, not an error:
2228                // nothing here failed.
2229                self.state.status = RunStatus::VerifiedNoop;
2230                self.state.save()?;
2231                self.settle_questions();
2232                return Ok(());
2233            }
2234            self.state.status = RunStatus::Failed;
2235            self.state.save()?;
2236            self.settle_questions();
2237            bail!("no candidate produced a change; nothing to judge");
2238        }
2239        self.state.status = RunStatus::Judging;
2240        self.state.save()?;
2241        Ok(())
2242    }
2243
2244    // --------------------------------------------------------------- judge
2245
2246    async fn judge(&mut self) -> Result<()> {
2247        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2248        // agent files with `magi task add` name the run that paid for it. The
2249        // prompt overlay is cloned alongside it because the waves borrow it
2250        // while `self` is mutably borrowed by the node's own bookkeeping.
2251        let run_id = self.state.id.clone();
2252        let prompts = self.state.config.prompts.clone();
2253        if !self.state.judgements.is_empty() || self.state.judge_skipped {
2254            return Ok(());
2255        }
2256        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2257        if viable.len() == 1 {
2258            // Recorded so this is a one-time event: `judgements` stays empty
2259            // either way, which without this flag is indistinguishable from
2260            // "not yet judged" on the next reentry — and status is left
2261            // untouched, so a later node's conclusion (e.g. `Blocked` after
2262            // the review budget ran out) survives a resume instead of being
2263            // clobbered back to `Judging` by this node running again.
2264            self.state.judge_skipped = true;
2265            self.state.event(
2266                "judge",
2267                format!(
2268                    "only candidate {} produced a change; judging skipped",
2269                    viable[0].label
2270                ),
2271            );
2272            self.state.save()?;
2273            return Ok(());
2274        }
2275        self.state.status = RunStatus::Judging;
2276
2277        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2278        let language = self.state.config.graph.language.clone();
2279        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2280        let sessions = self.state.config.graph.sessions;
2281        let artifacts = agent::artifacts_dir(&self.state.dir());
2282        let root = self.state.worktree_root();
2283        let base_short = short(&self.state.base_commit);
2284
2285        let mut jobs = Vec::new();
2286        let mut orders = Vec::new();
2287        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2288            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2289            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2290            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2291            let seat_key = format!("judge-{}", j + 1);
2292            let seat = self.seat(&seat_key, &spec.id);
2293            jobs.push(SeatJob {
2294                prompt: prompt::judge(
2295                    &self.state.instruction,
2296                    &views,
2297                    self.roles.judges.len(),
2298                    &base_short,
2299                    &language,
2300                ),
2301                spec,
2302                seat,
2303                cwd: root.join(format!("judge-{}", j + 1)),
2304                timeout,
2305                allow_write: false,
2306                sessions,
2307                artifacts: artifacts.clone(),
2308                stem: format!("judge-{}", j + 1),
2309            });
2310        }
2311
2312        self.state.event(
2313            "judge",
2314            format!(
2315                "{} judges ranking {} candidates blind",
2316                jobs.len(),
2317                viable.len()
2318            ),
2319        );
2320        let labels_for_check = labels.clone();
2321        let mut quota_losses = Vec::new();
2322        let cache = self.state.config.cache_dir();
2323        let ctx = WaveCtx {
2324            run: &run_id,
2325            node: "judge",
2326            prompts: &prompts,
2327            cache: cache.as_deref(),
2328            round: None,
2329        };
2330        let results = ask_json_wave::<Ranking>(
2331            jobs,
2332            Arc::clone(&self.sem),
2333            self.state.config.graph.retries,
2334            &ctx,
2335            &mut quota_losses,
2336            &mut self.state,
2337            &move |r: &Ranking| r.validate(&labels_for_check),
2338        )
2339        .await;
2340        self.state.quota.extend(quota_losses);
2341
2342        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2343            let agent_id = seat.agent.clone();
2344            self.state.seats.insert(seat.key.clone(), seat);
2345            let mut record = Judgement {
2346                judge: j + 1,
2347                seat: format!("judge-{}", j + 1),
2348                agent: agent_id,
2349                ranking: Vec::new(),
2350                reasons: BTreeMap::new(),
2351                confidence: None,
2352                order: orders[j].clone(),
2353                failed: None,
2354                duration_ms: 0,
2355            };
2356            match res {
2357                Ok((ranking, out)) => {
2358                    record.ranking = ranking.normalized();
2359                    record.reasons = ranking.reasons;
2360                    record.confidence = ranking.confidence;
2361                    record.duration_ms = out.duration_ms;
2362                    self.state.event(
2363                        "judge",
2364                        format!(
2365                            "judge {} ranked {}",
2366                            j + 1,
2367                            record.ranking.iter().collect::<String>()
2368                        ),
2369                    );
2370                }
2371                Err(e) => {
2372                    record.failed = Some(e.to_string());
2373                    self.state
2374                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2375                }
2376            }
2377            self.state.judgements.push(record);
2378            self.state.save()?;
2379        }
2380        Ok(())
2381    }
2382
2383    // ---------------------------------------------------------- deliberate
2384
2385    async fn deliberate(&mut self) -> Result<()> {
2386        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2387        // agent files with `magi task add` name the run that paid for it. The
2388        // prompt overlay is cloned alongside it because the waves borrow it
2389        // while `self` is mutably borrowed by the node's own bookkeeping.
2390        let run_id = self.state.id.clone();
2391        let prompts = self.state.config.prompts.clone();
2392        if !self.state.deliberation.is_empty() {
2393            return Ok(());
2394        }
2395        let tops: Vec<char> = self
2396            .state
2397            .judgements
2398            .iter()
2399            .filter_map(|j| j.ranking.first().copied())
2400            .collect();
2401        let rounds = self.state.config.graph.deliberate_rounds;
2402        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2403            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2404                self.state.event(
2405                    "deliberate",
2406                    format!("judges agreed on {} outright; no deliberation", tops[0]),
2407                );
2408            }
2409            self.state.status = RunStatus::Voting;
2410            self.state.save()?;
2411            return Ok(());
2412        }
2413
2414        self.state.status = RunStatus::Deliberating;
2415        self.state.event(
2416            "deliberate",
2417            format!(
2418                "split: first choices were {} — opening {rounds} round(s)",
2419                tops.iter().collect::<String>()
2420            ),
2421        );
2422
2423        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2424        let language = self.state.config.graph.language.clone();
2425        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2426        let sessions = self.state.config.graph.sessions;
2427        let artifacts = agent::artifacts_dir(&self.state.dir());
2428        let root = self.state.worktree_root();
2429        let base_short = short(&self.state.base_commit);
2430
2431        // Judges argue in sequence so that a turn can answer the one before it;
2432        // that is the difference between deliberation and three parallel
2433        // monologues.
2434        for round in 1..=rounds {
2435            let mut turns: Vec<DeliberationTurn> = Vec::new();
2436            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2437                if self.state.judgements[j].failed.is_some() {
2438                    continue;
2439                }
2440                let seat_key = format!("judge-{}", j + 1);
2441                let mut seat = self.seat(&seat_key, &spec.id);
2442                let transcript = self.transcript(&turns, j);
2443                let context = if has_context(&spec, &seat, sessions) {
2444                    None
2445                } else {
2446                    Some(self.candidate_block(&viable, &base_short))
2447                };
2448                let text = prompt::deliberate(
2449                    &self.state.instruction,
2450                    context.as_deref(),
2451                    &transcript,
2452                    round,
2453                    rounds,
2454                    &language,
2455                );
2456                let job = SeatJob {
2457                    spec,
2458                    seat: seat.clone(),
2459                    prompt: text,
2460                    cwd: root.join(format!("judge-{}", j + 1)),
2461                    timeout,
2462                    allow_write: false,
2463                    sessions,
2464                    artifacts: artifacts.clone(),
2465                    stem: format!("delib-{round}-judge-{}", j + 1),
2466                };
2467                let cache = self.state.config.cache_dir();
2468                let ctx = WaveCtx {
2469                    run: &run_id,
2470                    node: "deliberate",
2471                    prompts: &prompts,
2472                    cache: cache.as_deref(),
2473                    round: None,
2474                };
2475                let (updated, out) =
2476                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2477                seat = updated;
2478                let agent_id = seat.agent.clone();
2479                let seat_key = seat.key.clone();
2480                self.state.seats.insert(seat.key.clone(), seat);
2481                let body = match out {
2482                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2483                    // Never read the CLI's raw error JSON as this judge's
2484                    // position — skip the seat instead, the same as any other
2485                    // failed turn.
2486                    AgentOutcome::Dropped(o) => {
2487                        let why =
2488                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2489                                "the CLI ended the stream without delivering its answer",
2490                            );
2491                        self.state.event(
2492                            "deliberate",
2493                            format!(
2494                                "judge {} skipped: the CLI dropped the stream ({why})",
2495                                j + 1
2496                            ),
2497                        );
2498                        continue;
2499                    }
2500                    AgentOutcome::Quota(o) => {
2501                        self.state.quota.push(QuotaLoss {
2502                            seat: seat_key,
2503                            node: "deliberate".to_owned(),
2504                            at: Timestamp::now(),
2505                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2506                        });
2507                        self.state.event(
2508                            "deliberate",
2509                            format!("judge {} skipped: rate limited (quota)", j + 1),
2510                        );
2511                        continue;
2512                    }
2513                    AgentOutcome::Failed(e) => {
2514                        self.state
2515                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2516                        continue;
2517                    }
2518                };
2519                let tentative = verdict::extract_json::<Position>(&body)
2520                    .ok()
2521                    .and_then(|p| p.tentative)
2522                    .and_then(|s| s.trim().chars().next())
2523                    .map(|c| c.to_ascii_uppercase());
2524                self.state.event(
2525                    "deliberate",
2526                    format!(
2527                        "round {round}: judge {} now favours {}",
2528                        j + 1,
2529                        tentative.map_or("—".to_owned(), |c| c.to_string())
2530                    ),
2531                );
2532                turns.push(DeliberationTurn {
2533                    judge: j + 1,
2534                    agent: agent_id,
2535                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2536                    tentative,
2537                });
2538            }
2539            self.state
2540                .deliberation
2541                .push(DeliberationRound { round, turns });
2542            self.state.save()?;
2543        }
2544
2545        self.state.status = RunStatus::Voting;
2546        self.state.save()?;
2547        Ok(())
2548    }
2549
2550    // ---------------------------------------------------------------- vote
2551
2552    async fn vote(&mut self) -> Result<()> {
2553        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2554        // agent files with `magi task add` name the run that paid for it. The
2555        // prompt overlay is cloned alongside it because the waves borrow it
2556        // while `self` is mutably borrowed by the node's own bookkeeping.
2557        let run_id = self.state.id.clone();
2558        let prompts = self.state.config.prompts.clone();
2559        if !self.state.votes.is_empty() {
2560            return Ok(());
2561        }
2562        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2563        if viable.len() == 1 {
2564            return Ok(());
2565        }
2566        self.state.status = RunStatus::Voting;
2567
2568        let language = self.state.config.graph.language.clone();
2569        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2570        let sessions = self.state.config.graph.sessions;
2571        let artifacts = agent::artifacts_dir(&self.state.dir());
2572        let root = self.state.worktree_root();
2573        let base_short = short(&self.state.base_commit);
2574        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2575
2576        let mut jobs = Vec::new();
2577        let mut seats_at = Vec::new();
2578        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2579            if self
2580                .state
2581                .judgements
2582                .get(j)
2583                .is_some_and(|r| r.failed.is_some())
2584            {
2585                continue;
2586            }
2587            let seat_key = format!("judge-{}", j + 1);
2588            let seat = self.seat(&seat_key, &spec.id);
2589            let mut text = prompt::final_vote(&viable, &language);
2590            if !has_context(&spec, &seat, sessions) {
2591                text = format!(
2592                    "{}\n\n# Candidates\n\n{}",
2593                    text,
2594                    self.candidate_block(&candidates, &base_short)
2595                );
2596            }
2597            jobs.push(SeatJob {
2598                spec,
2599                seat,
2600                prompt: text,
2601                cwd: root.join(format!("judge-{}", j + 1)),
2602                timeout,
2603                allow_write: false,
2604                sessions,
2605                artifacts: artifacts.clone(),
2606                stem: format!("vote-judge-{}", j + 1),
2607            });
2608            seats_at.push(j);
2609        }
2610
2611        self.state.event(
2612            "vote",
2613            format!(
2614                "collecting {} final votes one by one, privately",
2615                jobs.len()
2616            ),
2617        );
2618        let allowed = viable.clone();
2619        let mut quota_losses = Vec::new();
2620        let cache = self.state.config.cache_dir();
2621        let ctx = WaveCtx {
2622            run: &run_id,
2623            node: "vote",
2624            prompts: &prompts,
2625            cache: cache.as_deref(),
2626            round: None,
2627        };
2628        let results = ask_json_wave::<FinalVote>(
2629            jobs,
2630            Arc::clone(&self.sem),
2631            self.state.config.graph.retries,
2632            &ctx,
2633            &mut quota_losses,
2634            &mut self.state,
2635            &move |v: &FinalVote| match v.label() {
2636                Some(c) if allowed.contains(&c) => Ok(()),
2637                other => bail!("vote {other:?} is not one of {allowed:?}"),
2638            },
2639        )
2640        .await;
2641        self.state.quota.extend(quota_losses);
2642
2643        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2644            let agent_id = seat.agent.clone();
2645            self.state.seats.insert(seat.key.clone(), seat);
2646            let initial = self
2647                .state
2648                .judgements
2649                .get(j)
2650                .and_then(|r| r.ranking.first().copied());
2651            let mut record = VoteRecord {
2652                judge: j + 1,
2653                agent: agent_id,
2654                vote: None,
2655                reason: String::new(),
2656                changed: false,
2657            };
2658            match res {
2659                Ok((v, _)) => {
2660                    record.vote = v.label();
2661                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2662                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2663                    self.state.event(
2664                        "vote",
2665                        format!(
2666                            "judge {} voted {}{}",
2667                            j + 1,
2668                            record.vote.unwrap_or('?'),
2669                            if record.changed { " (changed)" } else { "" }
2670                        ),
2671                    );
2672                }
2673                Err(e) => {
2674                    self.state
2675                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2676                }
2677            }
2678            self.state.votes.push(record);
2679            self.state.save()?;
2680        }
2681        Ok(())
2682    }
2683
2684    // --------------------------------------------------------------- tally
2685
2686    fn tally(&mut self) -> Result<()> {
2687        if self.state.tally.is_some() {
2688            return Ok(());
2689        }
2690        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2691        let tops: Vec<char> = self
2692            .state
2693            .judgements
2694            .iter()
2695            .filter_map(|j| j.ranking.first().copied())
2696            .collect();
2697        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2698
2699        // A judge whose private vote failed still counted once, in the initial
2700        // ranking; using it beats discarding a whole seat.
2701        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2702        let mut cast: Vec<char> = Vec::new();
2703        for (i, j) in self.state.judgements.iter().enumerate() {
2704            let vote = self
2705                .state
2706                .votes
2707                .iter()
2708                .find(|v| v.judge == i + 1)
2709                .and_then(|v| v.vote)
2710                .or_else(|| j.ranking.first().copied());
2711            if let Some(v) = vote {
2712                *first_choice.entry(v).or_insert(0) += 1;
2713                cast.push(v);
2714            }
2715        }
2716
2717        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2718        for j in &self.state.judgements {
2719            let n = j.ranking.len();
2720            for (pos, label) in j.ranking.iter().enumerate() {
2721                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2722            }
2723        }
2724
2725        let best = first_choice.values().copied().max().unwrap_or(0);
2726        let mut leaders: Vec<char> = first_choice
2727            .iter()
2728            .filter(|(_, v)| **v == best)
2729            .map(|(k, _)| *k)
2730            .collect();
2731        let mut tie_break = None;
2732        if leaders.len() > 1 {
2733            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2734            let borda_leaders: Vec<char> = leaders
2735                .iter()
2736                .copied()
2737                .filter(|l| borda[l] == top_borda)
2738                .collect();
2739            tie_break = Some(if borda_leaders.len() == 1 {
2740                format!(
2741                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2742                    leaders.len()
2743                )
2744            } else {
2745                format!(
2746                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2747                    leaders.len()
2748                )
2749            });
2750            leaders = borda_leaders;
2751            leaders.sort_unstable();
2752        }
2753        let winner = *leaders
2754            .first()
2755            .or(viable.first())
2756            .context("no candidate to declare a winner from")?;
2757
2758        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2759        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2760        let deliberated = !self.state.deliberation.is_empty();
2761
2762        // Whose verdict is this? A rate-limited seat is absent even if it
2763        // ranked before the limit hit, so presence is measured against the
2764        // recorded losses, not just "did a ranking ever appear".
2765        let quota_seats: std::collections::BTreeSet<&str> =
2766            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2767        let mut present = 0usize;
2768        for (i, j) in self.state.judgements.iter().enumerate() {
2769            if quota_seats.contains(j.seat.as_str()) {
2770                continue;
2771            }
2772            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2773            let voted = self
2774                .state
2775                .votes
2776                .iter()
2777                .any(|v| v.judge == i + 1 && v.vote.is_some());
2778            if ranked || voted {
2779                present += 1;
2780            }
2781        }
2782        // Strict majority of the configured panel. A bare majority is real
2783        // signal we can act on, while a minority verdict must never stand in
2784        // for a healthy one. A one-candidate run needs no panel at all, and
2785        // `judges` stays `0` rather than the roster size a panel that never
2786        // sat would otherwise be credited with.
2787        let needs_quorum = viable.len() > 1;
2788        let judges_total = if needs_quorum {
2789            self.roles.judges.len()
2790        } else {
2791            0
2792        };
2793        let quorum = if needs_quorum {
2794            judges_total / 2 + 1
2795        } else {
2796            0
2797        };
2798        let met_quorum = !needs_quorum || present >= quorum;
2799        let uncontested = (!needs_quorum).then(|| {
2800            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2801        });
2802
2803        self.state.event(
2804            "tally",
2805            match &uncontested {
2806                Some(reason) => format!("winner {winner} — {reason}"),
2807                None => format!(
2808                    "winner {winner} — votes {} | initial {} | {} changed | \
2809                     {present}/{judges_total} judges{}",
2810                    first_choice
2811                        .iter()
2812                        .map(|(k, v)| format!("{k}:{v}"))
2813                        .collect::<Vec<_>>()
2814                        .join(" "),
2815                    if unanimous_initial {
2816                        "unanimous"
2817                    } else {
2818                        "split"
2819                    },
2820                    changed_votes,
2821                    if met_quorum {
2822                        String::new()
2823                    } else {
2824                        format!(" — below quorum ({quorum} required)")
2825                    },
2826                ),
2827            },
2828        );
2829        if !met_quorum {
2830            self.state.event(
2831                "stall",
2832                format!(
2833                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2834                     the run stops here, resumable"
2835                ),
2836            );
2837        }
2838        self.state.tally = Some(Tally {
2839            first_choice,
2840            borda,
2841            winner,
2842            rankings: tops.len(),
2843            unanimous_initial,
2844            deliberated,
2845            changed_votes,
2846            unanimous_final,
2847            tie_break,
2848            judges: judges_total,
2849            present,
2850            quorum,
2851            met_quorum,
2852            uncontested,
2853        });
2854        self.state.status = if met_quorum {
2855            RunStatus::Reviewing
2856        } else {
2857            RunStatus::Stalled
2858        };
2859        self.state.save()?;
2860        Ok(())
2861    }
2862
2863    // ------------------------------------------------------------- recover
2864
2865    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2866    /// resumed toward completion once the transient cause clears.
2867    ///
2868    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2869    /// it toward the quorum, which is exactly the set of seats whose absence
2870    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2871    /// not depend on which node happened to hit the limit), or an ordinary
2872    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2873    /// seat is never disturbed.
2874    ///
2875    /// A seat that now answers with a usable ranking is "recovered": its
2876    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2877    /// `tally` counts it present again), and its vote re-collected. A seat that
2878    /// still fails keeps its loss and stays absent.
2879    ///
2880    /// Returns `true` when the re-tally restores the quorum (the run may proceed
2881    /// to review/gate/merge), `false` when it is still below quorum (the run
2882    /// stays `Stalled`, still resumable for a later retry).
2883    #[allow(clippy::too_many_lines)]
2884    async fn recover_stall(&mut self) -> Result<bool> {
2885        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2886        // agent files with `magi task add` name the run that paid for it. The
2887        // prompt overlay is cloned alongside it because the waves borrow it
2888        // while `self` is mutably borrowed by the node's own bookkeeping.
2889        let run_id = self.state.id.clone();
2890        let prompts = self.state.config.prompts.clone();
2891        // Absent seats = quota-lost at any node, or failed outright. Mirroring
2892        // `tally`'s presence test (rather than the old quota-judge/vote filter)
2893        // is what keeps a non-quota collapse — or a quota loss recorded at the
2894        // deliberate node — from being a permanent dead-end on `--resume`.
2895        let quota_seats: BTreeSet<&str> =
2896            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2897        let absent: Vec<String> = self
2898            .state
2899            .judgements
2900            .iter()
2901            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2902            .map(|j| j.seat.clone())
2903            .collect();
2904        if absent.is_empty() {
2905            return Ok(false);
2906        }
2907        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2908        if viable.len() <= 1 {
2909            return Ok(false);
2910        }
2911        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2912        let language = self.state.config.graph.language.clone();
2913        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2914        let sessions = self.state.config.graph.sessions;
2915        let artifacts = agent::artifacts_dir(&self.state.dir());
2916        let root = self.state.worktree_root();
2917        let base_short = short(&self.state.base_commit);
2918        let candidates: Vec<Candidate> = viable.clone();
2919
2920        // Map each absent seat key to its 0-based position in `roles.judges`.
2921        let mut positions: Vec<usize> = absent
2922            .iter()
2923            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2924            .collect();
2925        if positions.is_empty() {
2926            return Ok(false);
2927        }
2928        positions.sort_unstable();
2929        positions.dedup();
2930
2931        // Re-rank the lost seats, one blind prompt each.
2932        let mut judge_jobs = Vec::new();
2933        for &j in &positions {
2934            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2935            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2936            let seat_key = format!("judge-{}", j + 1);
2937            let spec = self.roles.judges[j].clone();
2938            let seat = self.seat(&seat_key, &spec.id);
2939            judge_jobs.push(SeatJob {
2940                spec,
2941                seat,
2942                prompt: prompt::judge(
2943                    &self.state.instruction,
2944                    &views,
2945                    self.roles.judges.len(),
2946                    &base_short,
2947                    &language,
2948                ),
2949                cwd: root.join(seat_key),
2950                timeout,
2951                allow_write: false,
2952                sessions,
2953                artifacts: artifacts.clone(),
2954                stem: format!("judge-{}-recover", j + 1),
2955            });
2956        }
2957
2958        let labels_for_check = labels.clone();
2959        let mut judge_losses = Vec::new();
2960        let retries = self.state.config.graph.retries;
2961        let cache = self.state.config.cache_dir();
2962        let ctx = WaveCtx {
2963            run: &run_id,
2964            node: "judge",
2965            prompts: &prompts,
2966            cache: cache.as_deref(),
2967            round: None,
2968        };
2969        let results = ask_json_wave::<Ranking>(
2970            judge_jobs,
2971            Arc::clone(&self.sem),
2972            retries,
2973            &ctx,
2974            &mut judge_losses,
2975            &mut self.state,
2976            &move |r: &Ranking| r.validate(&labels_for_check),
2977        )
2978        .await;
2979
2980        // Refresh the judgement of every seat that ranked again.
2981        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2982        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
2983            self.state.seats.insert(seat.key.clone(), seat);
2984            let record = &mut self.state.judgements[j];
2985            match res {
2986                Ok((ranking, out)) => {
2987                    record.ranking = ranking.normalized();
2988                    record.reasons = ranking.reasons;
2989                    record.confidence = ranking.confidence;
2990                    record.failed = None;
2991                    record.duration_ms = out.duration_ms;
2992                    recovered.insert(j);
2993                    self.state.event(
2994                        "recover",
2995                        format!("judge {} ranked again after the limit", j + 1),
2996                    );
2997                }
2998                Err(e) => {
2999                    self.state
3000                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3001                }
3002            }
3003        }
3004
3005        // Re-ask the votes of the seats that recovered a ranking.
3006        let mut vote_jobs = Vec::new();
3007        let mut vote_pos: Vec<usize> = Vec::new();
3008        for &j in &recovered {
3009            let seat_key = format!("judge-{}", j + 1);
3010            let spec = self.roles.judges[j].clone();
3011            let seat = self.seat(&seat_key, &spec.id);
3012            let mut text = prompt::final_vote(&labels, &language);
3013            if !has_context(&spec, &seat, sessions) {
3014                text = format!(
3015                    "{}\n\n# Candidates\n\n{}",
3016                    text,
3017                    self.candidate_block(&candidates, &base_short)
3018                );
3019            }
3020            vote_jobs.push(SeatJob {
3021                spec,
3022                seat,
3023                prompt: text,
3024                cwd: root.join(seat_key),
3025                timeout,
3026                allow_write: false,
3027                sessions,
3028                artifacts: artifacts.clone(),
3029                stem: format!("vote-judge-{}-recover", j + 1),
3030            });
3031            vote_pos.push(j);
3032        }
3033        let allowed = labels.clone();
3034        let mut vote_losses = Vec::new();
3035        let vote_retries = self.state.config.graph.retries;
3036        let vote_cache = self.state.config.cache_dir();
3037        let ctx = WaveCtx {
3038            run: &run_id,
3039            node: "vote",
3040            prompts: &prompts,
3041            cache: vote_cache.as_deref(),
3042            round: None,
3043        };
3044        let votes = ask_json_wave::<FinalVote>(
3045            vote_jobs,
3046            Arc::clone(&self.sem),
3047            vote_retries,
3048            &ctx,
3049            &mut vote_losses,
3050            &mut self.state,
3051            &move |v: &FinalVote| match v.label() {
3052                Some(c) if allowed.contains(&c) => Ok(()),
3053                other => bail!("vote {other:?} is not one of {allowed:?}"),
3054            },
3055        )
3056        .await;
3057        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3058            let agent_id = seat.agent.clone();
3059            self.state.seats.insert(seat.key.clone(), seat);
3060            match res {
3061                Ok((v, _)) => {
3062                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3063                        rec.vote = v.label();
3064                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3065                    } else {
3066                        self.state.votes.push(VoteRecord {
3067                            judge: j + 1,
3068                            agent: agent_id,
3069                            vote: v.label(),
3070                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3071                            changed: false,
3072                        });
3073                    }
3074                    self.state.event(
3075                        "recover",
3076                        format!("judge {} voted again after the limit", j + 1),
3077                    );
3078                }
3079                Err(e) => {
3080                    self.state
3081                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3082                }
3083            }
3084        }
3085
3086        // A seat that ranked again is present even if its re-vote failed —
3087        // `tally` falls back to the initial ranking's first choice — so clear
3088        // its quota loss. Seats that still fail keep theirs and stay absent.
3089        let recovered_keys: BTreeSet<String> = recovered
3090            .iter()
3091            .map(|&j| format!("judge-{}", j + 1))
3092            .collect();
3093        self.state
3094            .quota
3095            .retain(|q| !recovered_keys.contains(&q.seat));
3096        // A seat that hit the limit again is a fresh loss, not the old one:
3097        // replace the stale entry so the history stays one-per-seat and the
3098        // daemon can tell this attempt's loss from a previous session's.
3099        for loss in judge_losses.into_iter().chain(vote_losses) {
3100            if recovered_keys.contains(&loss.seat) {
3101                continue;
3102            }
3103            self.state.quota.retain(|q| q.seat != loss.seat);
3104            self.state.quota.push(loss);
3105        }
3106
3107        // Recompute the verdict from the refreshed panel.
3108        self.state.tally = None;
3109        self.tally()?;
3110        Ok(self
3111            .state
3112            .tally
3113            .as_ref()
3114            .map(|t| t.met_quorum)
3115            .unwrap_or(false))
3116    }
3117
3118    // ----------------------------------------------------------------- fold
3119
3120    async fn fold_losers(&mut self) -> Result<()> {
3121        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3122            return Ok(());
3123        };
3124        let repo = self.state.repo.clone();
3125        let mut folded = Vec::new();
3126        for i in 0..self.state.candidates.len() {
3127            let c = &self.state.candidates[i];
3128            if c.label == winner || c.folded {
3129                continue;
3130            }
3131            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3132            git::worktree_remove(&repo, &wt).await.ok();
3133            git::branch_delete(&repo, &branch).await.ok();
3134            self.state.candidates[i].folded = true;
3135            folded.push(label.to_string());
3136        }
3137        // The judges are finished; their checkouts are pure cost from here.
3138        let root = self.state.worktree_root();
3139        for j in 1..=self.roles.judges.len() {
3140            let wt = root.join(format!("judge-{j}"));
3141            if wt.exists() {
3142                git::worktree_remove(&repo, &wt).await.ok();
3143            }
3144        }
3145        // The design-deliberation stage is finished by the time a tally
3146        // exists — same reasoning as the judges above.
3147        if self.state.config.graph.advise {
3148            for k in 1..=self.state.config.graph.advisors {
3149                let wt = root.join(format!("advisor-{k}"));
3150                if wt.exists() {
3151                    git::worktree_remove(&repo, &wt).await.ok();
3152                }
3153            }
3154        }
3155        if !folded.is_empty() {
3156            self.state
3157                .event("fold", format!("folded candidates {}", folded.join(", ")));
3158            self.state.save()?;
3159        }
3160        Ok(())
3161    }
3162
3163    // ------------------------------------------------------------ base sync
3164
3165    /// Land the winner's tree on the current tip of `<remote>/<base>` before
3166    /// anything verifies it.
3167    ///
3168    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
3169    /// read whatever is checked out in the winner's worktree. Left alone that
3170    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
3171    /// it when the run *branched* - and a run takes long enough that the base
3172    /// has usually moved by the time it gets here. A gate that ran there
3173    /// answers "green on the commit this run started from", not "green on
3174    /// what is about to land", and the difference showed up three times in
3175    /// one day as a green run whose merge would have reverted a file another
3176    /// pull request had already landed.
3177    ///
3178    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
3179    /// implementation of the same idea: `land::Step::Rebase` already worked
3180    /// out the rules - throwaway worktree, conflict stops and reports rather
3181    /// than feeding a fixer, nothing runs in the primary tree - and a second
3182    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
3183    /// warns about ("two answers to a question nobody notices until a diff is
3184    /// wrong").
3185    ///
3186    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
3187    /// so it survives a park/resume. A conflict or a push failure sets
3188    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
3189    /// as they were - untouched, for a person to look at - which is also what
3190    /// makes re-entering this function afterwards a no-op instead of a second
3191    /// attempt at the same wall.
3192    async fn sync_to_base(&mut self) -> Result<()> {
3193        if self
3194            .state
3195            .base_sync
3196            .as_ref()
3197            .is_some_and(|s| s.conflict.is_some())
3198        {
3199            return Ok(());
3200        }
3201        let Some(winner) = self.state.winner().cloned() else {
3202            return Ok(());
3203        };
3204
3205        let repo = self.state.repo.clone();
3206        let remote = self.state.config.merge.remote.clone();
3207        let base_branch = self.state.base_branch.clone();
3208        let tracking = format!("{remote}/{base_branch}");
3209
3210        git::fetch(&repo, &remote, &base_branch).await.ok();
3211        // No network, or the remote never had this branch: `resolve_base`
3212        // already treats that as non-fatal at branch time, and a run that got
3213        // this far must not be blocked by it here either.
3214        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3215            return Ok(());
3216        };
3217
3218        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3219        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3220        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3221
3222        if behind == 0 {
3223            self.state.base_sync = Some(BaseSync {
3224                tip,
3225                behind: 0,
3226                attempts,
3227                conflict: None,
3228            });
3229            self.state.save()?;
3230            return Ok(());
3231        }
3232
3233        if attempts >= BASE_SYNC_ROUNDS {
3234            let why = format!(
3235                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3236                 rebase(s); rebasing again would only race it",
3237                winner.branch
3238            );
3239            self.state.status = RunStatus::Blocked;
3240            self.state.base_sync = Some(BaseSync {
3241                tip,
3242                behind,
3243                attempts,
3244                conflict: Some(why.clone()),
3245            });
3246            self.state.event("land", why);
3247            self.state.save()?;
3248            return Ok(());
3249        }
3250
3251        self.state.event(
3252            "land",
3253            format!(
3254                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3255                winner.branch
3256            ),
3257        );
3258        self.state.save()?;
3259
3260        let scratch = self.state.dir().join("base-sync");
3261        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
3262        let attempts = attempts + 1;
3263        match rebased {
3264            Ok(None) => {
3265                // The branch ref moved, but a worktree that already had it
3266                // checked out (the winner's) was not told; sync its index and
3267                // files before anything reads them.
3268                git::sync_to_head(&winner.worktree).await?;
3269                self.state.base_sync = Some(BaseSync {
3270                    tip: tip.clone(),
3271                    behind: 0,
3272                    attempts,
3273                    conflict: None,
3274                });
3275                self.state
3276                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
3277            }
3278            Ok(Some(conflict)) => {
3279                let why = format!(
3280                    "{} conflicts with {tracking} and did not rebase: {}",
3281                    winner.branch,
3282                    conflict.chars().take(600).collect::<String>()
3283                );
3284                self.state.status = RunStatus::Blocked;
3285                self.state.base_sync = Some(BaseSync {
3286                    tip,
3287                    behind,
3288                    attempts,
3289                    conflict: Some(why.clone()),
3290                });
3291                self.state.event("land", why);
3292            }
3293            Err(e) => {
3294                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3295                self.state.status = RunStatus::Blocked;
3296                self.state.base_sync = Some(BaseSync {
3297                    tip,
3298                    behind,
3299                    attempts,
3300                    conflict: Some(why.clone()),
3301                });
3302                self.state.event("land", why);
3303            }
3304        }
3305        self.state.save()?;
3306        Ok(())
3307    }
3308
3309    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
3310    /// last landed the winner on, once it has run, else the commit the run
3311    /// branched from.
3312    ///
3313    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
3314    /// and `vote` all happen before there is a winner to rebase, so they
3315    /// compare every candidate against the branch point on purpose, and a
3316    /// base that moves after they are already done cannot change an answer
3317    /// they already gave.
3318    fn landing_base(&self) -> String {
3319        self.state
3320            .base_sync
3321            .as_ref()
3322            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3323    }
3324
3325    // ------------------------------------------------------- operator fix
3326
3327    /// Route specific, already-recorded review findings to a fixer for a
3328    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
3329    /// entry point.
3330    ///
3331    /// Distinct from `review_loop`'s own fix step in three ways: it never
3332    /// runs a reviewer wave, it never spends review-round budget, and what
3333    /// happened is recorded as an [`OperatorFixRequest`] appended to
3334    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
3335    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
3336    /// and vote must never be rewritten to look like a manufactured blocking
3337    /// verdict.
3338    ///
3339    /// Only meaningful once review has actually concluded: `Ready` (handed
3340    /// off with findings still open, or simply concluded clean while minor
3341    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
3342    /// gate failed). Everything else is refused: a run still in progress
3343    /// should simply be resumed, and a `Merged` run's branch has already
3344    /// landed — reopening *this* run's own record cannot change that, so the
3345    /// answer there is a fresh `magi review <branch>`.
3346    ///
3347    /// A real commit here re-verifies through a fresh, ordinary review-only
3348    /// run on the same branch ([`Self::review`]) rather than reopening this
3349    /// run's own `review_loop`: once any round in this run's history went
3350    /// clean, `review_conclusion` treats that as permanent by design (the
3351    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
3352    /// way to force one more genuine reviewer wave out of *this* run without
3353    /// either rewriting history or weakening that guarantee for every other
3354    /// caller. A review-only run costs nothing extra — no implementation, no
3355    /// judging, no vote — and exercises the exact same review → verify →
3356    /// gate → (human) merge path, unmodified.
3357    pub async fn fix_selected(
3358        &mut self,
3359        ids: &[String],
3360        reason: &str,
3361        allow_stale: bool,
3362    ) -> Result<()> {
3363        let reason = reason.trim();
3364        if reason.is_empty() {
3365            bail!("a fix request needs a reason — that is the operator's own record of why");
3366        }
3367        if ids.is_empty() {
3368            bail!("no finding id given");
3369        }
3370        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3371            bail!(
3372                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3373                 has already concluded — can be given a targeted fix. A run still \
3374                 in progress should simply be resumed; a `merged` run's branch has \
3375                 already landed, so its answer is a fresh `magi review <branch>`, \
3376                 not reopening this run's own record",
3377                self.state.id,
3378                self.state.status.as_str()
3379            );
3380        }
3381        let Some(winner) = self.state.winner().cloned() else {
3382            bail!("run {} has no winning candidate to fix", self.state.id);
3383        };
3384        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3385            bail!(
3386                "branch `{}` no longer exists; this run cannot be extended",
3387                winner.branch
3388            );
3389        }
3390        let home = crate::run::home();
3391        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3392            bail!(
3393                "run {} is currently being worked on by another magi process",
3394                self.state.id
3395            );
3396        }
3397        // Held for the rest of this call, including the follow-up review
3398        // below: two `magi fix` invocations against the same run must not
3399        // both reach the worktree manipulation further down, which would
3400        // otherwise race to remove and recreate the same directory — see
3401        // [`FixClaim`]'s own doc.
3402        let _claim = FixClaim::acquire(&self.state.dir())?;
3403
3404        // Resolve every id before spending anything — an unknown id refuses
3405        // the whole request rather than silently dropping it — and dedup
3406        // while keeping the operator's own order.
3407        let mut seen = BTreeSet::new();
3408        let mut findings = Vec::new();
3409        let mut missing = Vec::new();
3410        for id in ids {
3411            if !seen.insert(id.clone()) {
3412                continue;
3413            }
3414            match self.state.finding(id) {
3415                Some((round, rec, f)) => findings.push(OperatorFixFinding {
3416                    id: f.id.clone(),
3417                    severity: f.severity,
3418                    reviewer_vote: rec.vote,
3419                    round: round.round,
3420                    round_head: round.head.clone(),
3421                    reviewer: rec.reviewer,
3422                    agent: rec.agent.clone(),
3423                    file: f.file.clone(),
3424                    line: f.line,
3425                    title: f.title.clone(),
3426                    detail: f.detail.clone(),
3427                    outcome: OperatorFixOutcome::Pending,
3428                }),
3429                None => missing.push(id.clone()),
3430            }
3431        }
3432        if !missing.is_empty() {
3433            bail!(
3434                "unknown finding id(s): {}; nothing was changed",
3435                missing.join(", ")
3436            );
3437        }
3438
3439        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3440        let stale_details: Vec<(String, String)> = findings
3441            .iter()
3442            .filter(|f| f.round_head != head_at_request)
3443            .map(|f| (f.id.clone(), f.round_head.clone()))
3444            .collect();
3445        let stale = !stale_details.is_empty();
3446        if stale && !allow_stale {
3447            bail!(
3448                "the branch has moved since some finding(s) were raised — {} — now \
3449                 at {}; pass --allow-stale to fix anyway, or re-run review first",
3450                stale_details
3451                    .iter()
3452                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3453                    .collect::<Vec<_>>()
3454                    .join(", "),
3455                short(&head_at_request)
3456            );
3457        }
3458
3459        let request = OperatorFixRequest {
3460            requested_at: Timestamp::now(),
3461            reason: reason.to_owned(),
3462            findings,
3463            head_at_request: head_at_request.clone(),
3464            allow_stale,
3465            stale,
3466            fix: None,
3467            result_head: None,
3468            follow_up_review_run: None,
3469        };
3470        self.state.event(
3471            "fix",
3472            format!(
3473                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3474                request.findings.len(),
3475                request
3476                    .findings
3477                    .iter()
3478                    .map(|f| f.id.as_str())
3479                    .collect::<Vec<_>>()
3480                    .join(", "),
3481            ),
3482        );
3483        // Recorded now, before any worktree work or the fixer call itself —
3484        // and re-saved at each checkpoint below: a crash at any point after
3485        // this (mid fixer call, mid follow-up review) must not lose the fact
3486        // that this was requested, for which findings, and why. Everything
3487        // past this point reads and writes through `request_index` rather
3488        // than a local variable, since `request` itself is moved here.
3489        self.state.operator_fixes.push(request);
3490        self.state.save()?;
3491        let request_index = self.state.operator_fixes.len() - 1;
3492
3493        // A fresh, dedicated worktree for this one call, never the winner's
3494        // own worktree in place: that one may already be gone (folded away),
3495        // and reusing it in place would leave the branch checked out there
3496        // when the follow-up review below tries to check it out again. Freed
3497        // immediately after, either way — but only once confirmed clean:
3498        // `worktree_remove` is a `git worktree remove --force`, which would
3499        // otherwise discard uncommitted work left there by the operator or
3500        // another process before this had a chance to even look at it.
3501        if winner.worktree.exists() {
3502            // Lockfiles a rescue commit withheld stay untracked on purpose and
3503            // are already recorded; they are not the operator's work to protect.
3504            let dirty = git::git(
3505                &winner.worktree,
3506                &["status", "--porcelain", "--untracked-files=all"],
3507            )
3508            .await?;
3509            let only_withheld = dirty.lines().all(|l| {
3510                l.strip_prefix("?? ")
3511                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3512            });
3513            if !only_withheld {
3514                bail!(
3515                    "`{}` has uncommitted changes; refusing to touch it — commit or \
3516                     discard them first",
3517                    winner.worktree.display()
3518                );
3519            }
3520            git::worktree_remove(&self.state.repo, &winner.worktree)
3521                .await
3522                .ok();
3523        }
3524        let fix_worktree = self.state.worktree_root().join("operator-fix");
3525        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3526        git::git(
3527            &self.state.repo,
3528            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3529        )
3530        .await
3531        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3532        if !git::is_clean(&fix_worktree).await? {
3533            git::worktree_remove(&self.state.repo, &fix_worktree)
3534                .await
3535                .ok();
3536            bail!(
3537                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3538                winner.branch
3539            );
3540        }
3541
3542        let run_id = self.state.id.clone();
3543        let prompts = self.state.config.prompts.clone();
3544        let language = self.state.config.graph.language.clone();
3545        let sessions = self.state.config.graph.sessions;
3546        let artifacts = agent::artifacts_dir(&self.state.dir());
3547        let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3548            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3549            _ => (
3550                self.state
3551                    .config
3552                    .agent(&winner.agent)
3553                    .cloned()
3554                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3555                format!("impl-{}", winner.label),
3556            ),
3557        };
3558        let seat = self.seat(&fix_seat_key, &fix_spec.id);
3559        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3560            .findings
3561            .iter()
3562            .map(|f| Finding {
3563                id: f.id.clone(),
3564                severity: f.severity,
3565                file: f.file.clone(),
3566                line: f.line,
3567                title: f.title.clone(),
3568                detail: f.detail.clone(),
3569            })
3570            .collect();
3571        let job = SeatJob {
3572            prompt: prompt::operator_fix(
3573                &self.state.instruction,
3574                &finding_list,
3575                reason,
3576                &stale_details,
3577                &head_at_request,
3578                &language,
3579            ),
3580            spec: fix_spec.clone(),
3581            seat,
3582            cwd: fix_worktree.clone(),
3583            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3584            allow_write: true,
3585            sessions,
3586            artifacts: artifacts.clone(),
3587            stem: "operator-fix".to_owned(),
3588        };
3589        let cache = self.state.config.cache_dir();
3590        let ctx = WaveCtx {
3591            run: &run_id,
3592            node: "fix",
3593            prompts: &prompts,
3594            cache: cache.as_deref(),
3595            round: None,
3596        };
3597        let (seat, out) =
3598            run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3599        let agent_id = seat.agent.clone();
3600
3601        let mut fix = FixRecord {
3602            agent: agent_id,
3603            addressed: Vec::new(),
3604            rejected: Vec::new(),
3605            notes: String::new(),
3606            committed: false,
3607            failed: None,
3608            duration_ms: 0,
3609            continuation: None,
3610        };
3611        let mut final_seat = seat.clone();
3612        match out {
3613            AgentOutcome::Ok(o) => {
3614                fix.duration_ms = o.duration_ms;
3615                let parsed = verdict::extract_json::<FixReport>(&o.text);
3616                let incomplete_reason = match &parsed {
3617                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3618                        "the reply parsed, but it reported a command whose own CLI \
3619                         never confirmed an exit status"
3620                            .to_owned(),
3621                    ),
3622                    Ok(_) => None,
3623                    Err(e) => Some(e.to_string()),
3624                };
3625                match incomplete_reason {
3626                    None => {
3627                        let report = parsed.expect("checked Ok above");
3628                        fix.addressed = report.addressed;
3629                        fix.rejected = report.rejected;
3630                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
3631                    }
3632                    Some(reason) => {
3633                        let (resumed_seat, resolved, failure, cont) = self
3634                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
3635                            .await;
3636                        fix.duration_ms += cont.cumulative_wait_ms;
3637                        fix.continuation = Some(cont);
3638                        final_seat = resumed_seat;
3639                        match resolved {
3640                            Some(report) => {
3641                                fix.addressed = report.addressed;
3642                                fix.rejected = report.rejected;
3643                                fix.notes =
3644                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
3645                            }
3646                            None => fix.failed = failure,
3647                        }
3648                    }
3649                }
3650            }
3651            AgentOutcome::Dropped(o) => {
3652                fix.duration_ms = o.duration_ms;
3653                let why = o
3654                    .dropped
3655                    .as_ref()
3656                    .map(|d| d.why.as_str())
3657                    .unwrap_or("the CLI ended the stream without delivering its answer");
3658                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3659            }
3660            AgentOutcome::Quota(o) => {
3661                self.state.quota.push(QuotaLoss {
3662                    seat: final_seat.key.clone(),
3663                    node: "fix".to_owned(),
3664                    at: Timestamp::now(),
3665                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3666                });
3667                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3668            }
3669            AgentOutcome::Failed(e) => fix.failed = Some(e),
3670        }
3671        if fix.continuation.is_none() {
3672            fix.continuation = Some(ContinuationRecord::not_needed());
3673        }
3674        self.state.seats.insert(final_seat.key.clone(), final_seat);
3675
3676        let rescue_message = format!(
3677            "magi: operator-selected fix ({}) (uncommitted work)",
3678            self.state.operator_fixes[request_index]
3679                .findings
3680                .iter()
3681                .map(|f| f.id.as_str())
3682                .collect::<Vec<_>>()
3683                .join(", ")
3684        );
3685        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
3686            self.state.note_withheld("fix", &r.withheld);
3687        }
3688        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
3689        fix.committed = after != head_at_request;
3690        git::worktree_remove(&self.state.repo, &fix_worktree)
3691            .await
3692            .ok();
3693
3694        self.state.event(
3695            "fix",
3696            match &fix.failed {
3697                Some(reason) => format!(
3698                    "operator fix: adoption report was lost ({reason}); {}",
3699                    if fix.committed {
3700                        "committed"
3701                    } else {
3702                        "NO new commit"
3703                    }
3704                ),
3705                None => format!(
3706                    "operator fix: {} addressed, {} rejected, {}",
3707                    fix.addressed.len(),
3708                    fix.rejected.len(),
3709                    if fix.committed {
3710                        "committed"
3711                    } else {
3712                        "NO new commit"
3713                    }
3714                ),
3715            },
3716        );
3717
3718        // Every selected finding gets an outcome — never left `Pending` once
3719        // the fixer's own turn is over. A report that never came back at all
3720        // marks every one of them `Unreported`, not silently "not addressed":
3721        // quota, a dropped stream, or an exhausted continuation are gaps in
3722        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
3723        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
3724        for f in &mut self.state.operator_fixes[request_index].findings {
3725            f.outcome = if fix.failed.is_some() {
3726                OperatorFixOutcome::Unreported
3727            } else if fix.addressed.contains(&f.id) {
3728                OperatorFixOutcome::Addressed
3729            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
3730                OperatorFixOutcome::Rejected { why: r.why.clone() }
3731            } else {
3732                OperatorFixOutcome::Unreported
3733            };
3734        }
3735
3736        let committed = fix.committed;
3737        if committed {
3738            self.state.operator_fixes[request_index].result_head = Some(after.clone());
3739        }
3740        self.state.operator_fixes[request_index].fix = Some(fix);
3741        // Saved again now that the fixer's own outcome is final, on top of
3742        // the save right after the request was first pushed above.
3743        self.state.save()?;
3744
3745        if committed {
3746            self.state.event(
3747                "fix",
3748                format!(
3749                    "operator fix committed {}; opening a follow-up review-only run",
3750                    short(&after)
3751                ),
3752            );
3753            match Self::review(&self.state.repo, &winner.branch, self.state.config.clone()).await {
3754                Ok(mut follow_up) => {
3755                    follow_up.state.event(
3756                        "start",
3757                        format!(
3758                            "requested by an operator fix on run {} for finding(s) {}",
3759                            self.state.id,
3760                            self.state.operator_fixes[request_index]
3761                                .findings
3762                                .iter()
3763                                .map(|f| f.id.as_str())
3764                                .collect::<Vec<_>>()
3765                                .join(", "),
3766                        ),
3767                    );
3768                    follow_up.state.save()?;
3769                    let follow_up_id = follow_up.state.id.clone();
3770                    if let Err(e) = follow_up.execute().await {
3771                        self.state.event(
3772                            "fix",
3773                            format!(
3774                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
3775                            ),
3776                        );
3777                    }
3778                    self.state.operator_fixes[request_index].follow_up_review_run =
3779                        Some(follow_up_id);
3780                }
3781                Err(e) => {
3782                    self.state.event(
3783                        "fix",
3784                        format!("committed the fix but could not open a follow-up review: {e:#}"),
3785                    );
3786                }
3787            }
3788            self.state.save()?;
3789        }
3790
3791        Ok(())
3792    }
3793
3794    // --------------------------------------------------------------- review
3795
3796    /// The agent and seat key that fix the winner's tree: the configured
3797    /// fixer, else the winner's own implementer seat, whose conversation
3798    /// continues now that the competition is over. Shared by the review loop
3799    /// and the gate-fix round so both talk to the same seat.
3800    fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
3801        match &self.roles.fixer {
3802            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3803            _ => (
3804                self.state
3805                    .config
3806                    .agent(&winner.agent)
3807                    .cloned()
3808                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3809                format!("impl-{}", winner.label),
3810            ),
3811        }
3812    }
3813
3814    async fn review_loop(&mut self) -> Result<()> {
3815        // A base that would not rebase is a person's decision, not a review
3816        // round: nothing here would change the answer, and reviewers and a
3817        // fixer would be spending real budget on a tree that cannot land
3818        // regardless of what they find.
3819        if self
3820            .state
3821            .base_sync
3822            .as_ref()
3823            .is_some_and(|s| s.conflict.is_some())
3824        {
3825            return Ok(());
3826        }
3827        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3828        // agent files with `magi task add` name the run that paid for it. The
3829        // prompt overlay is cloned alongside it because the waves borrow it
3830        // while `self` is mutably borrowed by the node's own bookkeeping.
3831        let run_id = self.state.id.clone();
3832        let prompts = self.state.config.prompts.clone();
3833        let Some(winner) = self.state.winner().cloned() else {
3834            return Ok(());
3835        };
3836        let max_rounds = self.state.config.graph.review_rounds;
3837        // A clean round, an exhausted round budget, or a stalled tree (see
3838        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
3839        // they are recorded — recomputed here, not read off `status`, so a
3840        // reentry into a run that already stopped restates the identical
3841        // verdict instead of silently handing back whatever an earlier node
3842        // in this same walk clobbered `status` to (a solo-candidate
3843        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
3844        // below runs an empty range once the budget is spent, and would
3845        // otherwise fall through without touching `status` at all.
3846        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
3847            self.state.status = status;
3848            self.state.save()?;
3849            return Ok(());
3850        }
3851        self.state.status = RunStatus::Reviewing;
3852        // A last recorded round whose own verification never resolved
3853        // (`ResourceBlocked` — the shared build cache, not the patch) is
3854        // never a concluded round, whatever the round budget says: starting
3855        // a fresh round on top of it would spend a whole new reviewer wave
3856        // re-reading an unchanged patch instead of just retrying the one
3857        // check that actually needs it, and once the budget is spent the
3858        // loop below has nothing left to do at all (its range is empty).
3859        // Retry that check directly instead, exactly the same retry
3860        // `stop_reviewing` already does for its own catch-up case.
3861        if self
3862            .state
3863            .reviews
3864            .last()
3865            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
3866        {
3867            let shell = self.state.config.shell();
3868            return self
3869                .stop_reviewing(
3870                    "the last round's own verification never resolved",
3871                    &shell,
3872                    &winner.worktree,
3873                )
3874                .await;
3875        }
3876
3877        let repo = self.state.repo.clone();
3878        let root = self.state.worktree_root();
3879        let language = self.state.config.graph.language.clone();
3880        let sessions = self.state.config.graph.sessions;
3881        let artifacts = agent::artifacts_dir(&self.state.dir());
3882        let base = self.landing_base();
3883        let base_short = short(&base);
3884        let reviewers = self.roles.reviewers.clone();
3885        let shell = self.state.config.shell();
3886
3887        for round in (self.state.reviews.len() + 1)..=max_rounds {
3888            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3889            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
3890            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
3891            // The prior round's own record, already persisted — never a
3892            // hand-carried variable of just its failing output: that is
3893            // exactly what let a round's e2e result drift out of sync with
3894            // which commit it was actually about (see `SCHEMA`'s doc for
3895            // schema 8). Judged against `head`, the commit reviewers are
3896            // about to look at now, so the summary always reads as "an
3897            // earlier head" here — this round's own patch has not been
3898            // checked yet.
3899            let prev_verification = self
3900                .state
3901                .reviews
3902                .last()
3903                .and_then(|r| r.verification_summary(&head));
3904
3905            // Each reviewer gets its own detached checkout of exactly this
3906            // commit: nobody can perturb the winner's tree, and the fixer can
3907            // keep working without racing a reviewer.
3908            let mut jobs = Vec::new();
3909            for (r, spec) in reviewers.iter().cloned().enumerate() {
3910                let wt = root.join(format!("review-{}", r + 1));
3911                if wt.exists() {
3912                    git::reset_detached(&wt, &head).await?;
3913                } else {
3914                    git::worktree_add_detached(&repo, &wt, &head).await?;
3915                }
3916                let seat_key = format!("review-{}", r + 1);
3917                let seat = self.seat(&seat_key, &spec.id);
3918                jobs.push(SeatJob {
3919                    prompt: prompt::review(&prompt::ReviewCtx {
3920                        instruction: &self.state.instruction,
3921                        branch: &winner.branch,
3922                        base_short: &base_short,
3923                        stat: &stat,
3924                        patch: &patch,
3925                        verification: prev_verification.as_ref(),
3926                        reviewers: reviewers.len(),
3927                        round,
3928                        rounds: max_rounds,
3929                        // A review-only run has no rankings, so nothing
3930                        // competed for this patch and the reviewer is told so.
3931                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
3932                        lens: Lens::for_seat(r),
3933                        language: &language,
3934                    }),
3935                    spec,
3936                    seat,
3937                    cwd: wt,
3938                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3939                    allow_write: false,
3940                    sessions,
3941                    artifacts: artifacts.clone(),
3942                    stem: format!("review-{round}-{}", r + 1),
3943                });
3944            }
3945
3946            self.state.event(
3947                "review",
3948                format!(
3949                    "round {round}: {} reviewers on {}",
3950                    jobs.len(),
3951                    short(&head)
3952                ),
3953            );
3954            let mut quota_losses = Vec::new();
3955            let review_retries = self.state.config.graph.retries;
3956            let review_cache = self.state.config.cache_dir();
3957            let ctx = WaveCtx {
3958                run: &run_id,
3959                node: "review",
3960                prompts: &prompts,
3961                cache: review_cache.as_deref(),
3962                round: Some(round),
3963            };
3964            let results = ask_json_wave::<Review>(
3965                jobs,
3966                Arc::clone(&self.sem),
3967                review_retries,
3968                &ctx,
3969                &mut quota_losses,
3970                &mut self.state,
3971                &|_: &Review| Ok(()),
3972            )
3973            .await;
3974            // Counted before the move below: how many of *this* round's
3975            // reviewer seats were lost to their own rate limit, as opposed to
3976            // a crash, a timeout, or unparsable output — see `round_is_clean`.
3977            let round_quota_missing = quota_losses.len();
3978            self.state.quota.extend(quota_losses);
3979
3980            let mut records = Vec::new();
3981            let mut all_findings = Vec::new();
3982            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
3983                let agent_id = seat.agent.clone();
3984                self.state.seats.insert(seat.key.clone(), seat);
3985                let mut record = ReviewRecord {
3986                    reviewer: r + 1,
3987                    agent: agent_id,
3988                    summary: String::new(),
3989                    findings: Vec::new(),
3990                    vote: None,
3991                    failed: None,
3992                    duration_ms: 0,
3993                    // Set for both outcomes: `failed: Some(_)` with
3994                    // `attempts > 0` is a seat every retry still lost, not a
3995                    // recovered one — only `failed: None` with `attempts > 0`
3996                    // reads as "answered after a nudge" (see this field's own
3997                    // doc).
3998                    attempts,
3999                };
4000                match res {
4001                    Ok((review, out)) => {
4002                        // Sanitized here, at the point every other piece of
4003                        // agent prose in this file is (candidate summaries,
4004                        // deliberation turns, vote reasons): a reviewer's own
4005                        // words are the one thing about it that could name
4006                        // it, and reconsideration below broadcasts this same
4007                        // summary and these same findings to every other
4008                        // seat on the panel.
4009                        record.summary =
4010                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
4011                        record.vote = Some(review.vote);
4012                        record.duration_ms = out.duration_ms;
4013                        for (n, mut f) in review.findings.into_iter().enumerate() {
4014                            // ids are magi's, never the agent's: the fixer's
4015                            // adoption report is keyed by them.
4016                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4017                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4018                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4019                            // `file` is agent-supplied prose too, never
4020                            // checked against the real tree — the same
4021                            // exposure `title`/`detail` above have, just in
4022                            // a field easy to forget because it looks like a
4023                            // path rather than free text.
4024                            f.file = f
4025                                .file
4026                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4027                            all_findings.push(f.clone());
4028                            record.findings.push(f);
4029                        }
4030                        self.state.event(
4031                            "review",
4032                            format!(
4033                                "round {round}: reviewer {} voted {} with {} finding(s)",
4034                                r + 1,
4035                                review.vote.label(),
4036                                record.findings.len()
4037                            ),
4038                        );
4039                    }
4040                    Err(e) => {
4041                        record.failed = Some(e.to_string());
4042                        self.state.event(
4043                            "review",
4044                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4045                        );
4046                    }
4047                }
4048                records.push(record);
4049            }
4050
4051            // Tally the round's votes and, if they split, spend the one
4052            // round of reconsideration the split -> deliberate -> revote
4053            // shape `judge`/`vote` use for the panel, sized down to what a
4054            // read-only review round can afford: one round, and a revote
4055            // rather than an argument, because the panel already wrote its
4056            // reasoning down as findings the first time around.
4057            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4058            let vote_split =
4059                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4060            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4061            if vote_split {
4062                self.state.event(
4063                    "review",
4064                    format!(
4065                        "round {round}: votes split ({}) — one round of reconsideration",
4066                        initial_votes
4067                            .iter()
4068                            .map(|v| v.label())
4069                            .collect::<Vec<_>>()
4070                            .join(", ")
4071                    ),
4072                );
4073                // Seats read every seat's findings and votes, still numbered
4074                // and never named — the same anonymity `review` itself keeps.
4075                let panel: Vec<ReviewSeatReport<'_>> = records
4076                    .iter()
4077                    .filter_map(|r| {
4078                        r.vote.map(|vote| ReviewSeatReport {
4079                            reviewer: r.reviewer,
4080                            vote,
4081                            summary: &r.summary,
4082                            findings: &r.findings,
4083                        })
4084                    })
4085                    .collect();
4086
4087                let mut jobs = Vec::new();
4088                let mut seats_at = Vec::new();
4089                for (r, spec) in reviewers.iter().cloned().enumerate() {
4090                    // A seat with no initial vote has nothing to reconsider
4091                    // from and stays absent, the same as it stayed absent
4092                    // from `panel` above.
4093                    if records[r].vote.is_none() {
4094                        continue;
4095                    }
4096                    let wt = root.join(format!("review-{}", r + 1));
4097                    let seat_key = format!("review-{}", r + 1);
4098                    let seat = self.seat(&seat_key, &spec.id);
4099                    // A seat with no live session has already forgotten the
4100                    // initial review's prompt — restate the patch it is
4101                    // voting on, the same as `deliberate`/`vote` do for a
4102                    // judge in the same position.
4103                    let patch_ctx = if has_context(&spec, &seat, sessions) {
4104                        None
4105                    } else {
4106                        Some(ReviewPatch {
4107                            branch: &winner.branch,
4108                            base_short: &base_short,
4109                            stat: &stat,
4110                            patch: &patch,
4111                        })
4112                    };
4113                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
4114                        instruction: &self.state.instruction,
4115                        reviewer: r + 1,
4116                        lens: Lens::for_seat(r),
4117                        panel: &panel,
4118                        patch: patch_ctx,
4119                        round,
4120                        rounds: max_rounds,
4121                        language: &language,
4122                    });
4123                    jobs.push(SeatJob {
4124                        prompt,
4125                        spec,
4126                        seat,
4127                        cwd: wt,
4128                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4129                        allow_write: false,
4130                        sessions,
4131                        artifacts: artifacts.clone(),
4132                        stem: format!("review-{round}-reconsider-{}", r + 1),
4133                    });
4134                    seats_at.push(r);
4135                }
4136
4137                let mut recon_quota_losses = Vec::new();
4138                let recon_cache = self.state.config.cache_dir();
4139                let recon_ctx = WaveCtx {
4140                    run: &run_id,
4141                    node: "review",
4142                    prompts: &prompts,
4143                    cache: recon_cache.as_deref(),
4144                    round: Some(round),
4145                };
4146                let recon_results = ask_json_wave::<ReviewRevote>(
4147                    jobs,
4148                    Arc::clone(&self.sem),
4149                    review_retries,
4150                    &recon_ctx,
4151                    &mut recon_quota_losses,
4152                    &mut self.state,
4153                    &|_: &ReviewRevote| Ok(()),
4154                )
4155                .await;
4156                self.state.quota.extend(recon_quota_losses);
4157
4158                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4159                    let agent_id = seat.agent.clone();
4160                    self.state.seats.insert(seat.key.clone(), seat);
4161                    let mut rec = ReviewRevoteRecord {
4162                        reviewer: r + 1,
4163                        agent: agent_id,
4164                        vote: None,
4165                        reason: String::new(),
4166                        failed: None,
4167                    };
4168                    match res {
4169                        Ok((rv, _)) => {
4170                            rec.vote = Some(rv.vote);
4171                            rec.reason =
4172                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4173                            self.state.event(
4174                                "review",
4175                                format!(
4176                                    "round {round}: reviewer {} revoted {}",
4177                                    r + 1,
4178                                    rv.vote.label()
4179                                ),
4180                            );
4181                        }
4182                        Err(e) => {
4183                            rec.failed = Some(e.to_string());
4184                            self.state.event(
4185                                "review",
4186                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4187                            );
4188                        }
4189                    }
4190                    reconsideration.push(rec);
4191                }
4192            } else if initial_votes.len() > 1 {
4193                self.state.event(
4194                    "review",
4195                    format!(
4196                        "round {round}: votes agreed ({}) — no reconsideration",
4197                        initial_votes[0].label()
4198                    ),
4199                );
4200            }
4201
4202            // The final vote per seat is its revote where reconsideration
4203            // ran and answered, its initial vote otherwise — the same
4204            // fallback `tally` uses for a judge whose private vote failed.
4205            let final_votes: Vec<ReviewVote> = records
4206                .iter()
4207                .filter_map(|r| {
4208                    reconsideration
4209                        .iter()
4210                        .find(|rv| rv.reviewer == r.reviewer)
4211                        .and_then(|rv| rv.vote)
4212                        .or(r.vote)
4213                })
4214                .collect();
4215            let round_verdict = ReviewVote::worst(final_votes);
4216
4217            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4218            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4219            // A round that already has a blocking finding and a round left to
4220            // try is going back to the fixer no matter what `verify.e2e`
4221            // says, so running it first only spends the loop's slowest step
4222            // (minutes, for a Rust repo's full test suite) on a head about
4223            // to be rewritten. Deferred, never skipped: `verify.e2e` still
4224            // runs once a round has no blocking findings left (see
4225            // `round_is_clean`, which a deferred — empty — `e2e` can never
4226            // satisfy since `blocking` is nonzero whenever this branch is
4227            // taken), and `stop_reviewing` forces a real run before it will
4228            // ever read a deferred round as green.
4229            let defer_e2e =
4230                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4231            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4232                let reason =
4233                    format!("{blocking} blocking finding(s) already required a fix this round");
4234                self.state.event(
4235                    "verify",
4236                    format!(
4237                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4238                         {}); it will run once a round has none left",
4239                        short(&head)
4240                    ),
4241                );
4242                (Vec::new(), false, true, Some(reason))
4243            } else {
4244                let e2e_commands = self.state.config.verify.e2e.clone();
4245                let cache_dir = self.state.config.cache_dir();
4246                let context = format!("round {round}");
4247                let (e2e, verify_retried) = with_cache_lease(
4248                    &mut self.state,
4249                    cache_dir.as_deref(),
4250                    "e2e",
4251                    "e2e",
4252                    &winner.worktree,
4253                    &head,
4254                    verify_timeout,
4255                    &context,
4256                    |state, budget| {
4257                        let shell = shell.clone();
4258                        let e2e_commands = e2e_commands.clone();
4259                        let worktree = winner.worktree.clone();
4260                        let context = context.clone();
4261                        async move {
4262                            run_e2e_with_retry(
4263                                state,
4264                                &shell,
4265                                &e2e_commands,
4266                                &worktree,
4267                                budget,
4268                                &context,
4269                            )
4270                            .await
4271                        }
4272                    },
4273                )
4274                .await;
4275                (e2e, verify_retried, false, None)
4276            };
4277
4278            let expected = records.len();
4279            let answered = records.iter().filter(|r| r.failed.is_none()).count();
4280            let incomplete = answered < expected;
4281            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4282            let policy = self.state.config.graph.incomplete_review;
4283            let clean = round_is_clean(
4284                blocking,
4285                e2e_ok,
4286                answered,
4287                expected,
4288                round_quota_missing,
4289                policy,
4290            );
4291
4292            let mut round_record = ReviewRound {
4293                round,
4294                head: head.clone(),
4295                verified_head: None,
4296                verified_at: None,
4297                reviews: records,
4298                e2e,
4299                verify_retried,
4300                e2e_deferred,
4301                e2e_defer_reason,
4302                fix: None,
4303                blocking,
4304                answered,
4305                expected,
4306                clean,
4307                progressed: false,
4308                vote_split,
4309                reconsideration,
4310                verdict: round_verdict,
4311            };
4312            // Which commit and when magi actually attempted to check —
4313            // known the moment a command was dispatched against `head`,
4314            // whether or not it finished: a resource-blocked attempt still
4315            // targeted a specific commit at a specific time, and leaving
4316            // that unrecorded is exactly what made `verification_summary`
4317            // report a fresh attempt as "commit unknown ... recorded before
4318            // this was tracked", indistinguishable from a genuinely old,
4319            // untracked record. Only a deferred or unconfigured round never
4320            // ran at all and has nothing to record — see
4321            // `ReviewRound::verified_head`'s own doc.
4322            if !matches!(
4323                round_record.e2e_status(),
4324                E2eStatus::Deferred | E2eStatus::NotConfigured
4325            ) {
4326                round_record.verified_head = Some(head.clone());
4327                round_record.verified_at = Some(Timestamp::now());
4328            }
4329            let this_round_verification = round_record.verification_summary(&head);
4330
4331            if incomplete {
4332                let missing: Vec<String> = round_record
4333                    .reviews
4334                    .iter()
4335                    .filter(|r| r.failed.is_some())
4336                    .map(|r| format!("review-{}", r.reviewer))
4337                    .collect();
4338                self.state.event(
4339                    "review",
4340                    format!(
4341                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4342                        missing.join(", ")
4343                    ),
4344                );
4345            }
4346
4347            if clean {
4348                self.state.event(
4349                    "review",
4350                    if incomplete && policy == IncompleteReviewPolicy::Warn {
4351                        format!(
4352                            "round {round}: clean (warn policy, incomplete panel) — no \
4353                             blocking findings from the seats that answered, verification green"
4354                        )
4355                    } else if incomplete {
4356                        format!(
4357                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4358                             quorum) — no blocking findings from the seats that answered, \
4359                             verification green",
4360                            expected - answered
4361                        )
4362                    } else {
4363                        format!("round {round}: clean — no blocking findings, verification green")
4364                    },
4365                );
4366                self.state.reviews.push(round_record);
4367                self.state.status = RunStatus::Gating;
4368                self.state.save()?;
4369                return Ok(());
4370            }
4371
4372            // Nothing was raised and verification passed, but not every seat
4373            // answered and `round_is_clean` still refused to call it clean —
4374            // either a seat is missing for a reason other than its own quota
4375            // (a crash, a timeout, unparsable output — worth another try), or
4376            // every seat that could have answered lost its quota and nobody
4377            // is left to decide on: re-review rather than send the fixer
4378            // after a round with nothing to fix.
4379            if incomplete && blocking == 0 && e2e_ok {
4380                self.state.reviews.push(round_record);
4381                self.state.save()?;
4382                if round == max_rounds {
4383                    self.state.status = RunStatus::Blocked;
4384                    self.state.event(
4385                        "review",
4386                        format!(
4387                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4388                             refusing to call it clean",
4389                            expected - answered
4390                        ),
4391                    );
4392                    return Ok(());
4393                }
4394                continue;
4395            }
4396
4397            // Nothing for the fixer to act on (`blocking == 0`) and the only
4398            // reason this round is not clean is that magi itself never got
4399            // a command to run — the shared build cache, not the patch (see
4400            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
4401            // the fixer would invite a change to appease contention that has
4402            // nothing to do with the diff, and would leave this attempt
4403            // sitting in the next round's prompt as if it were about an
4404            // earlier, superseded commit rather than what it actually is:
4405            // the same head, still waiting to be checked. Wait for it the
4406            // same way the final round's own contention is already handled,
4407            // whatever round this happens to be.
4408            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4409                self.state.reviews.push(round_record);
4410                return self
4411                    .stop_reviewing(
4412                        "the round's own verification could not run",
4413                        &shell,
4414                        &winner.worktree,
4415                    )
4416                    .await;
4417            }
4418
4419            if round == max_rounds {
4420                self.state.reviews.push(round_record);
4421                return self
4422                    .stop_reviewing(
4423                        &format!(
4424                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4425                        ),
4426                        &shell,
4427                        &winner.worktree,
4428                    )
4429                    .await;
4430            }
4431
4432            // Fix. The winner's own implementer seat continues its conversation:
4433            // the competition is over, so context is pure benefit now.
4434            let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4435            let seat = self.seat(&fix_seat_key, &fix_spec.id);
4436            let blocking_findings: Vec<_> = all_findings
4437                .iter()
4438                .filter(|f| f.severity.blocks())
4439                .cloned()
4440                .collect();
4441            let job = SeatJob {
4442                prompt: prompt::fix(
4443                    &self.state.instruction,
4444                    &blocking_findings,
4445                    this_round_verification.as_ref(),
4446                    round,
4447                    max_rounds,
4448                    &language,
4449                ),
4450                spec: fix_spec.clone(),
4451                seat,
4452                cwd: winner.worktree.clone(),
4453                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4454                allow_write: true,
4455                sessions,
4456                artifacts: artifacts.clone(),
4457                stem: format!("fix-{round}"),
4458            };
4459            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4460            let cache = self.state.config.cache_dir();
4461            let ctx = WaveCtx {
4462                run: &run_id,
4463                node: "fix",
4464                prompts: &prompts,
4465                cache: cache.as_deref(),
4466                round: Some(round),
4467            };
4468            let (seat, out) =
4469                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4470            let agent_id = seat.agent.clone();
4471
4472            let mut fix = FixRecord {
4473                agent: agent_id,
4474                addressed: Vec::new(),
4475                rejected: Vec::new(),
4476                notes: String::new(),
4477                committed: false,
4478                failed: None,
4479                duration_ms: 0,
4480                continuation: None,
4481            };
4482            let mut continuation = ContinuationRecord::not_needed();
4483            let mut final_seat = seat.clone();
4484            match out {
4485                AgentOutcome::Ok(o) => {
4486                    fix.duration_ms = o.duration_ms;
4487                    let parsed = verdict::extract_json::<FixReport>(&o.text);
4488                    // A parsed report standing next to a command this same
4489                    // reply's own CLI never confirmed the exit status of is
4490                    // not a resolved answer — the identical `CommandEvidence`
4491                    // `state.jobs` renders, read here instead of only on
4492                    // display, per the completion judgment and the shown
4493                    // record needing to agree.
4494                    let incomplete_reason = match &parsed {
4495                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4496                            "the reply parsed, but it reported a command whose own CLI never \
4497                             confirmed an exit status"
4498                                .to_owned(),
4499                        ),
4500                        Ok(_) => None,
4501                        Err(e) => Some(e.to_string()),
4502                    };
4503                    match incomplete_reason {
4504                        None => {
4505                            let report = parsed.expect("checked Ok above");
4506                            fix.addressed = report.addressed;
4507                            fix.rejected = report.rejected;
4508                            fix.notes =
4509                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
4510                        }
4511                        Some(reason) => {
4512                            let (resumed_seat, resolved, failure, cont) = self
4513                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4514                                .await;
4515                            fix.duration_ms += cont.cumulative_wait_ms;
4516                            continuation = cont;
4517                            final_seat = resumed_seat;
4518                            match resolved {
4519                                Some(report) => {
4520                                    fix.addressed = report.addressed;
4521                                    fix.rejected = report.rejected;
4522                                    fix.notes = blind::sanitize_prose(
4523                                        &report.notes,
4524                                        &self.state.config.blind,
4525                                    );
4526                                }
4527                                None => fix.failed = failure,
4528                            }
4529                        }
4530                    }
4531                }
4532                // The CLI's raw error JSON is not a fix report to parse.
4533                AgentOutcome::Dropped(o) => {
4534                    fix.duration_ms = o.duration_ms;
4535                    let why = o
4536                        .dropped
4537                        .as_ref()
4538                        .map(|d| d.why.as_str())
4539                        .unwrap_or("the CLI ended the stream without delivering its answer");
4540                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4541                }
4542                AgentOutcome::Quota(o) => {
4543                    self.state.quota.push(QuotaLoss {
4544                        seat: final_seat.key.clone(),
4545                        node: "fix".to_owned(),
4546                        at: Timestamp::now(),
4547                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4548                    });
4549                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4550                }
4551                AgentOutcome::Failed(e) => fix.failed = Some(e),
4552            }
4553            fix.continuation = Some(continuation);
4554            self.state.seats.insert(final_seat.key.clone(), final_seat);
4555            if let Ok(r) = git::rescue_commit(
4556                &winner.worktree,
4557                &format!("magi: review round {round} fixes (uncommitted work)"),
4558            )
4559            .await
4560            {
4561                self.state.note_withheld("fix", &r.withheld);
4562            }
4563            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
4564            fix.committed = after != before;
4565            // Judged by what `git` says moved against base, never by the
4566            // fixer's own `addressed`/`rejected` count — see
4567            // `ReviewRound::progressed`. Propagated with `?`, the same as the
4568            // `patch` snapshot above: swallowing this error would default
4569            // `diff_after` to empty, which almost always differs from a
4570            // non-empty `patch` and reads as "progressed" — exactly backwards
4571            // for a `git` failure the stagnation check cannot see through.
4572            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
4573            let progressed = diff_after != patch;
4574            let commit_note = if fix.committed {
4575                "committed"
4576            } else {
4577                "NO new commit"
4578            };
4579            let tree_note = if progressed {
4580                "changed vs base"
4581            } else {
4582                "unchanged vs base"
4583            };
4584            self.state.event(
4585                "fix",
4586                match &fix.failed {
4587                    // Distinct on purpose from "0 addressed, 0 rejected": the
4588                    // fixer's own diff still landed (blocking counts do keep
4589                    // falling round over round), only its adoption report did
4590                    // not come back, so this must never read like every
4591                    // finding was reviewed and declined.
4592                    Some(reason) => {
4593                        format!(
4594                            "round {round}: fixer's adoption report was lost ({reason}); \
4595                             {commit_note}, tree {tree_note}"
4596                        )
4597                    }
4598                    None => format!(
4599                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
4600                         {tree_note}{}",
4601                        fix.addressed.len(),
4602                        fix.rejected.len(),
4603                        if continuation.outcome == ContinuationOutcome::Resumed {
4604                            format!(
4605                                " (adoption report recovered after {} continuation(s))",
4606                                continuation.attempts
4607                            )
4608                        } else {
4609                            String::new()
4610                        },
4611                    ),
4612                },
4613            );
4614            round_record.fix = Some(fix);
4615            round_record.progressed = progressed;
4616            self.state.reviews.push(round_record);
4617            self.state.save()?;
4618
4619            // The fixer's own report never came back this round, even after
4620            // `continue_fix_report`'s own budget was spent on it — not an
4621            // ordinary "no report" (dropped stream, quota, plain failure),
4622            // which already reads that way and is left to the existing round
4623            // budget. Stopping here, rather than opening another round, is
4624            // what keeps a next reviewer/fixer wave from ever being
4625            // dispatched onto `winner.worktree` while whatever the seat's
4626            // last call may still have running there is unaccounted for: no
4627            // process liveness check exists (and none is being added — see
4628            // AGENTS.md/this task's own scope), so the only way to honour
4629            // "nothing starts before a valid report returns" is to not start
4630            // anything further on this worktree from this run at all.
4631            if matches!(
4632                continuation.outcome,
4633                ContinuationOutcome::Exhausted
4634                    | ContinuationOutcome::QuotaLost
4635                    | ContinuationOutcome::NoSession
4636            ) {
4637                return self
4638                    .stop_reviewing(
4639                        "the fixer's adoption report never came back, even after resuming its \
4640                         own seat; refusing to start another round against the same worktree \
4641                         while that is unresolved",
4642                        &shell,
4643                        &winner.worktree,
4644                    )
4645                    .await;
4646            }
4647
4648            let streak = self
4649                .state
4650                .reviews
4651                .iter()
4652                .rev()
4653                .take_while(|r| !r.progressed)
4654                .count();
4655            if streak >= STAGNANT_LIMIT {
4656                return self
4657                    .stop_reviewing(
4658                        &format!(
4659                            "the tree has not moved against base for {streak} round(s) in a row"
4660                        ),
4661                        &shell,
4662                        &winner.worktree,
4663                    )
4664                    .await;
4665            }
4666        }
4667        Ok(())
4668    }
4669
4670    /// Decide, from the last recorded round's own verification, whether
4671    /// stopping the review loop is a hand-off or a genuine block.
4672    ///
4673    /// Called once the loop has given up trying — the round budget is spent,
4674    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
4675    /// findings still open, never while a round is still clean or the
4676    /// incomplete-panel case handled inline above. Gate and e2e are facts
4677    /// about the tree; a lingering review finding is an opinion, and this
4678    /// workload's own `magi stats` puts reviewer precision low enough
4679    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
4680    /// must not by itself stand between a green, verified change and the
4681    /// human who decides what to do with it. A red e2e is not an opinion, so
4682    /// that case still blocks, with the failing command and a tail of its
4683    /// output recorded here rather than left in `run.json` for someone to go
4684    /// find.
4685    ///
4686    /// A round that deferred its own e2e (see [`Config::graph`]'s
4687    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
4688    /// only because nothing ran, and treating an empty list as a passing one
4689    /// here is exactly the "deferred painted green" bug this function exists
4690    /// to not have. When the last round's own verification never resolved —
4691    /// deferred on purpose, or a real attempt the shared build cache blocked
4692    /// — this makes (or retries) the real run, on the actual worktree this
4693    /// loop is about to stop touching, before deciding anything. A
4694    /// resource-blocked attempt is likewise never read as either green or
4695    /// red: it is evidence about the machine, not the patch (see
4696    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
4697    /// blocked cache leaves this call without deciding rather than guessing
4698    /// — the caller retries on a later reentry.
4699    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
4700        let round_idx = self.state.reviews.len() - 1;
4701        // A deferred round and a resource-blocked one are the same shape
4702        // here: neither has a real result yet, and both get one more
4703        // attempt. Read off `e2e_status` — the single source for this —
4704        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
4705        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
4706        // still retries instead of being read as a settled result the
4707        // instant it stops being empty.
4708        let needs_catchup_run = matches!(
4709            self.state.reviews[round_idx].e2e_status(),
4710            E2eStatus::Deferred | E2eStatus::ResourceBlocked
4711        );
4712        if needs_catchup_run {
4713            let round = self.state.reviews[round_idx].round;
4714            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4715            let commands = self.state.config.verify.e2e.clone();
4716            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
4717            let cache_dir = self.state.config.cache_dir();
4718            let context = format!(
4719                "round {round}: verification unresolved, catching up before the final decision"
4720            );
4721            let (outcomes, verify_retried) = with_cache_lease(
4722                &mut self.state,
4723                cache_dir.as_deref(),
4724                "e2e",
4725                "e2e",
4726                worktree,
4727                &attempted_head,
4728                timeout,
4729                &context,
4730                |state, budget| {
4731                    let shell = shell.to_vec();
4732                    let commands = commands.clone();
4733                    let context = context.clone();
4734                    async move {
4735                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
4736                            .await
4737                    }
4738                },
4739            )
4740            .await;
4741            let last = &mut self.state.reviews[round_idx];
4742            last.e2e = outcomes;
4743            last.verify_retried = verify_retried;
4744            // Always the commit and time this attempt actually targeted,
4745            // whether or not it happens to equal the reviewed `head` and
4746            // whether or not a command finished — see
4747            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
4748            // attempt is recorded too, so a later reader sees "attempted
4749            // again at T2" rather than silence.
4750            last.verified_head = Some(attempted_head);
4751            last.verified_at = Some(Timestamp::now());
4752            if verify_inconclusive(&last.e2e) {
4753                // Still not a real result: `e2e_deferred` is left exactly
4754                // as it was, so `needs_catchup_run` above reads
4755                // `ResourceBlocked` (via `e2e_status`, which checks
4756                // `resource_blocked` before `e2e_deferred`) and retries
4757                // again on the next reentry, rather than recording
4758                // contention as a red e2e and blocking the run on it.
4759                self.state.save()?;
4760                return Ok(());
4761            }
4762            last.e2e_deferred = false;
4763        }
4764        let last = &self.state.reviews[round_idx];
4765        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
4766
4767        match last.e2e_status() {
4768            E2eStatus::Failed => {
4769                let red: Vec<String> = last
4770                    .e2e
4771                    .iter()
4772                    .filter(|o| !o.ok())
4773                    .map(|o| {
4774                        format!(
4775                            "`{}` -> {:?}\n{}",
4776                            o.command,
4777                            o.code,
4778                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4779                        )
4780                    })
4781                    .collect();
4782                self.state
4783                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
4784                self.state.status = RunStatus::Blocked;
4785            }
4786            // `needs_catchup_run` above already retried once this call; if
4787            // it is still blocked, this is magi's own admission it could
4788            // not get a command to run, never a verdict on the patch — the
4789            // run is left exactly where a later reentry can retry again.
4790            E2eStatus::ResourceBlocked => {
4791                self.state.event(
4792                    "review",
4793                    format!(
4794                        "{why}; e2e could not run (shared build cache unavailable); not \
4795                         deciding yet"
4796                    ),
4797                );
4798            }
4799            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
4800                self.state.event(
4801                    "review",
4802                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
4803                );
4804                self.state.status = RunStatus::Gating;
4805            }
4806        }
4807        self.state.save()?;
4808        Ok(())
4809    }
4810
4811    // ----------------------------------------------------------------- gate
4812
4813    async fn gate(&mut self) -> Result<()> {
4814        // Judged by the review record itself, not by `status`: a solo
4815        // candidate's `judge`/`deliberate` skip rewrites `status` on every
4816        // reentry (see `judge`), and trusting it here is exactly how a run
4817        // that exhausted its review budget got gated and merged a second
4818        // time around. `review_conclusion` recomputes the review loop's own
4819        // verdict from the round records themselves — `Gating` for a clean
4820        // round or a hand-off (see `stop_reviewing`), anything else means the
4821        // loop is still going or genuinely blocked.
4822        // A base the winner could not be replayed onto is a decision, not a
4823        // round: there is no landing tree to gate. Read as its own record for
4824        // the same reason the review verdict is.
4825        if self.state.status == RunStatus::Failed
4826            || self
4827                .state
4828                .base_sync
4829                .as_ref()
4830                .is_some_and(|s| s.conflict.is_some())
4831            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4832                != Some(RunStatus::Gating)
4833        {
4834            return Ok(());
4835        }
4836        if self.state.gate_ran {
4837            // `review_loop` derives its conclusion from the clean review
4838            // record on every reentry and therefore puts a completed run back
4839            // in `Gating`. A recorded gate is a stronger, terminal fact:
4840            // retain its original command output (or lack of any, for a repo
4841            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
4842            // doc) and restore `Blocked` on a real failure rather than
4843            // pretending the command is still running or running it a second
4844            // time. `gate_ran == false` remains the only shape — unattempted,
4845            // or a resource-blocked retry — that may still need to execute a
4846            // command.
4847            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
4848                self.state.status = RunStatus::Blocked;
4849                self.state.save()?;
4850            }
4851            return Ok(());
4852        }
4853        let Some(winner) = self.state.winner().cloned() else {
4854            return Ok(());
4855        };
4856        self.state.status = RunStatus::Gating;
4857        let mut outcomes = self.run_gate(&winner).await?;
4858        loop {
4859            // A resource-blocked outcome means the gate command never actually
4860            // ran - the shared build cache could not be acquired or confirmed
4861            // fresh in time - which is evidence about the machine, not about
4862            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
4863            // Recording it as a red gate would mark a run `Blocked` on nothing
4864            // but contention magi has already logged; leaving `self.state.gate`
4865            // empty and `self.state.gate_ran` false instead keeps the shape
4866            // this function already treats as "still needs to run" (see the
4867            // early-return above), so the next call retries the command
4868            // rather than concluding anything.
4869            if verify_inconclusive(&outcomes) {
4870                self.state.save()?;
4871                return Ok(());
4872            }
4873            if outcomes.iter().all(CommandOutcome::ok) {
4874                break;
4875            }
4876            match self.gate_fix_round(&winner, &outcomes).await? {
4877                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
4878                GateFix::Stop => break,
4879                GateFix::Defer => {
4880                    self.state.save()?;
4881                    return Ok(());
4882                }
4883            }
4884        }
4885        let passed = outcomes.iter().all(CommandOutcome::ok);
4886        self.state.gate = outcomes;
4887        self.state.gate_ran = true;
4888        if !passed {
4889            self.state.status = RunStatus::Blocked;
4890            let spent = self.state.gate_fixes.len();
4891            self.state.event(
4892                "gate",
4893                if spent == 0 {
4894                    "gate failed; not merging".to_owned()
4895                } else {
4896                    format!("gate failed after {spent} gate-fix round(s); not merging")
4897                },
4898            );
4899        }
4900        self.state.save()?;
4901        Ok(())
4902    }
4903
4904    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
4905    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
4906    /// after review is clean and never on a candidate awaiting judging.
4907    ///
4908    /// Never fails the run: a non-zero exit or timeout is a warning and a
4909    /// recorded outcome, and the gate remains the single arbiter. Nothing
4910    /// configured means nothing happens - no event, no commit. `commit_all`
4911    /// commits any leftover change under the neutral identity and returns
4912    /// `false` when the tree is clean, so no empty commit is ever made.
4913    async fn run_pre_gate(&mut self, winner: &Candidate) {
4914        let commands = self.state.config.verify.pre_gate.clone();
4915        if commands.is_empty() {
4916            return;
4917        }
4918        let shell = self.state.config.shell();
4919        let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4920        let (outcomes, _) = run_commands(
4921            &mut self.state,
4922            "pre_gate",
4923            "pre_gate",
4924            0,
4925            &shell,
4926            &commands,
4927            &winner.worktree,
4928            timeout,
4929        )
4930        .await;
4931        for o in &outcomes {
4932            if !o.ok() {
4933                tracing::warn!(
4934                    "pre_gate `{}` failed ({:?}); the gate decides",
4935                    o.command,
4936                    o.code
4937                );
4938            }
4939            self.state.event(
4940                "pre_gate",
4941                format!(
4942                    "`{}` -> {}",
4943                    o.command,
4944                    if o.ok() {
4945                        "pass".to_owned()
4946                    } else {
4947                        format!(
4948                            "FAIL ({:?})\n{}",
4949                            o.code,
4950                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4951                        )
4952                    }
4953                ),
4954            );
4955        }
4956        self.state.pre_gate = outcomes;
4957        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
4958            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
4959                Ok(head) => {
4960                    self.state
4961                        .event("pre_gate", format!("committed mechanical fixes ({head})"));
4962                    self.state.pre_gate_commit = Some(head);
4963                }
4964                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
4965            },
4966            Ok(false) => {}
4967            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
4968        }
4969        if let Err(e) = self.state.save() {
4970            tracing::warn!("could not persist the pre_gate record: {e:#}");
4971        }
4972    }
4973
4974    /// Run `verify.gate` once against the winner's current tree, logging one
4975    /// event per command. Empty when nothing is configured.
4976    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
4977        self.run_pre_gate(winner).await;
4978        let shell = self.state.config.shell();
4979        let gate_commands = self.state.config.verify.gate.clone();
4980        // Zero commands has nothing to run and nothing that could touch the
4981        // shared build cache, so it never needs a lease: `Config::cache_dir`
4982        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
4983        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
4984        // otherwise queue behind an unrelated run's lease and come back
4985        // resource-blocked - `gate_ran` would stay false on nothing but
4986        // cache contention, for a step that had nothing to check in the
4987        // first place.
4988        let outcomes = if gate_commands.is_empty() {
4989            Vec::new()
4990        } else {
4991            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4992            let cache_dir = self.state.config.cache_dir();
4993            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4994            let (outcomes, _) = with_cache_lease(
4995                &mut self.state,
4996                cache_dir.as_deref(),
4997                "gate",
4998                "gate",
4999                &winner.worktree,
5000                &head,
5001                timeout,
5002                "final gate",
5003                |state, budget| {
5004                    let shell = shell.clone();
5005                    let gate_commands = gate_commands.clone();
5006                    let worktree = winner.worktree.clone();
5007                    async move {
5008                        let (outcomes, timed_out_pids) = run_commands(
5009                            state,
5010                            "gate",
5011                            "gate",
5012                            0,
5013                            &shell,
5014                            &gate_commands,
5015                            &worktree,
5016                            budget,
5017                        )
5018                        .await;
5019                        (outcomes, false, timed_out_pids)
5020                    }
5021                },
5022            )
5023            .await;
5024            outcomes
5025        };
5026        if outcomes.is_empty() {
5027            // Nothing configured to check — distinct from every other
5028            // silence in this run's event log, since an empty `gate` alone
5029            // no longer says whether the gate ran at all (see
5030            // `RunState::gate_ran`'s own doc).
5031            self.state.event(
5032                "gate",
5033                "no gate commands configured; nothing to check, passing",
5034            );
5035        }
5036        for o in &outcomes {
5037            self.state.event(
5038                "gate",
5039                format!(
5040                    "`{}` -> {}",
5041                    o.command,
5042                    if o.ok() {
5043                        "pass".to_owned()
5044                    } else {
5045                        format!(
5046                            "FAIL ({:?})\n{}",
5047                            o.code,
5048                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5049                        )
5050                    }
5051                ),
5052            );
5053        }
5054        Ok(outcomes)
5055    }
5056
5057    /// One bounded fix round for a failing gate.
5058    ///
5059    /// The fixer is told the failure came from the gate itself, not from a
5060    /// reviewer, and is shown the failed commands, their exit codes and a tail
5061    /// of their output - whatever `[verify].gate` holds, nothing here knows
5062    /// what those commands run. Only a normal non-zero exit that printed
5063    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
5064    /// command or a full disk says nothing about the code, and a fixer sent
5065    /// after it can only appease the machine. The round is judged by what git
5066    /// says moved, never by the fixer's own report, and `verify.e2e` runs
5067    /// again before the gate does, so a fix cannot trade a green gate for a
5068    /// red e2e unnoticed.
5069    async fn gate_fix_round(
5070        &mut self,
5071        winner: &Candidate,
5072        outcomes: &[CommandOutcome],
5073    ) -> Result<GateFix> {
5074        let cap = self.state.config.graph.gate_fix_rounds;
5075        let spent = self.state.gate_fixes.len();
5076        if spent >= cap {
5077            if cap > 0 {
5078                self.state.event(
5079                    "gate",
5080                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
5081                );
5082            }
5083            return Ok(GateFix::Stop);
5084        }
5085        if !gate_fixable(outcomes) {
5086            self.state.event(
5087                "gate",
5088                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5089                 command or similar); not spending a fix round on it",
5090            );
5091            return Ok(GateFix::Stop);
5092        }
5093        let min_free = self.state.config.disk.min_free_bytes;
5094        if min_free > 0 {
5095            match crate::disk::free_bytes(&winner.worktree) {
5096                Ok(free) if crate::disk::enough_space(free, min_free) => {}
5097                Ok(free) => {
5098                    self.state.event(
5099                        "gate",
5100                        format!(
5101                            "only {free} bytes free ({min_free} required by `[disk] \
5102                             min_free_bytes`); not spending a fix round on a failure the disk \
5103                             may explain"
5104                        ),
5105                    );
5106                    return Ok(GateFix::Stop);
5107                }
5108                Err(e) => {
5109                    self.state.event(
5110                        "gate",
5111                        format!("free disk space could not be measured ({e:#}); no fix round"),
5112                    );
5113                    return Ok(GateFix::Stop);
5114                }
5115            }
5116        }
5117
5118        let attempt = spent + 1;
5119        let run_id = self.state.id.clone();
5120        let prompts = self.state.config.prompts.clone();
5121        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5122        let base = self.landing_base();
5123        let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5124        let seat = self.seat(&fix_seat_key, &fix_spec.id);
5125        let job = SeatJob {
5126            prompt: prompt::gate_fix(
5127                &self.state.instruction,
5128                &failed,
5129                attempt,
5130                cap,
5131                &self.state.config.graph.language,
5132            ),
5133            spec: fix_spec,
5134            seat,
5135            cwd: winner.worktree.clone(),
5136            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5137            allow_write: true,
5138            sessions: self.state.config.graph.sessions,
5139            artifacts: agent::artifacts_dir(&self.state.dir()),
5140            stem: format!("gate-fix-{attempt}"),
5141        };
5142        self.state.event(
5143            "gate",
5144            format!("gate failed; gate-fix round {attempt} of {cap}"),
5145        );
5146        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5147        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5148        let cache = self.state.config.cache_dir();
5149        let ctx = WaveCtx {
5150            run: &run_id,
5151            node: "gate-fix",
5152            prompts: &prompts,
5153            cache: cache.as_deref(),
5154            round: None,
5155        };
5156        let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5157        let mut record = GateFixRecord {
5158            agent: seat.agent.clone(),
5159            failed,
5160            notes: String::new(),
5161            committed: false,
5162            error: None,
5163        };
5164        match out {
5165            AgentOutcome::Ok(o) => {
5166                // A missing report is not a failed fix: the round is judged
5167                // by the tree below, and the report only carries prose.
5168                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5169                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5170                }
5171            }
5172            AgentOutcome::Dropped(_) => {
5173                record.error = Some("the CLI dropped the stream".to_owned());
5174            }
5175            AgentOutcome::Quota(o) => {
5176                self.state.quota.push(QuotaLoss {
5177                    seat: seat.key.clone(),
5178                    node: "gate-fix".to_owned(),
5179                    at: Timestamp::now(),
5180                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5181                });
5182                record.error = Some("rate limited (quota); fixer could not run".to_owned());
5183            }
5184            AgentOutcome::Failed(e) => record.error = Some(e),
5185        }
5186        self.state.seats.insert(seat.key.clone(), seat);
5187        if let Ok(r) = git::rescue_commit(
5188            &winner.worktree,
5189            &format!("magi: gate fix {attempt} (uncommitted work)"),
5190        )
5191        .await
5192        {
5193            self.state.note_withheld("gate-fix", &r.withheld);
5194        }
5195        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5196        record.committed = after != before;
5197        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5198        let note = record.error.clone();
5199        self.state.gate_fixes.push(record);
5200        self.state.save()?;
5201        if !changed {
5202            self.state.event(
5203                "gate",
5204                match note {
5205                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5206                    None => format!("gate-fix round {attempt}: the tree did not change"),
5207                },
5208            );
5209            return Ok(GateFix::Stop);
5210        }
5211        self.state.event(
5212            "gate",
5213            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5214        );
5215
5216        let commands = self.state.config.verify.e2e.clone();
5217        if !commands.is_empty() {
5218            let shell = self.state.config.shell();
5219            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5220            let cache_dir = self.state.config.cache_dir();
5221            let context = format!("gate-fix round {attempt}");
5222            let (e2e, _) = with_cache_lease(
5223                &mut self.state,
5224                cache_dir.as_deref(),
5225                "e2e",
5226                "e2e",
5227                &winner.worktree,
5228                &after,
5229                timeout,
5230                &context,
5231                |state, budget| {
5232                    let shell = shell.clone();
5233                    let commands = commands.clone();
5234                    let context = context.clone();
5235                    let worktree = winner.worktree.clone();
5236                    async move {
5237                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5238                            .await
5239                    }
5240                },
5241            )
5242            .await;
5243            if verify_inconclusive(&e2e) {
5244                return Ok(GateFix::Defer);
5245            }
5246            if e2e.iter().any(|o| !o.ok()) {
5247                self.state.event(
5248                    "gate",
5249                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5250                );
5251                return Ok(GateFix::Stop);
5252            }
5253        }
5254        Ok(GateFix::Retry)
5255    }
5256
5257    // ---------------------------------------------------------------- merge
5258
5259    async fn merge(&mut self) -> Result<()> {
5260        // Same reasoning as `gate`: ask the review and gate records directly
5261        // rather than `status`, which a solo-candidate `judge`/`deliberate`
5262        // skip can rewrite on reentry to something that no longer says
5263        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
5264        // so a hand-off (open findings, green verification) reaches merge
5265        // exactly like a genuinely clean round does.
5266        //
5267        // A run resumed mid-`land` never reaches here at all: `execute`
5268        // recognises `RunStatus::Landing` before it even calls `prep`, and
5269        // routes straight to `run_land` instead. That has to happen a level
5270        // up from this function, not with a check in here, because
5271        // `review_loop`'s own status recomputation (see its doc) runs
5272        // *before* `merge` on every reentry and would otherwise overwrite
5273        // the `Landing` marker with `Gating` before this node ever saw it.
5274        if self
5275            .state
5276            .base_sync
5277            .as_ref()
5278            .is_some_and(|s| s.conflict.is_some())
5279            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5280                != Some(RunStatus::Gating)
5281            // `gate_ran == false` is not "passed" - `gate` leaves it false
5282            // both before it has ever run and when its last attempt was
5283            // resource-blocked (see `Runner::gate`'s own doc), and neither is
5284            // permission to merge on nothing but the review record. Only a
5285            // gate that actually ran - zero commands configured and
5286            // vacuously passed, or one or more that all exited 0 - may
5287            // proceed; `RunState::gate_status` is the single place that
5288            // reading is computed.
5289            || !self.state.gate_status().ok()
5290        {
5291            return Ok(());
5292        }
5293        // This node's own record, not `status`: `status == Ready` is not
5294        // unique to the harmless `MergeMode::None` path this line was
5295        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
5296        // run's PR was closed without merging — and on that run `mode` is
5297        // still `Pr`, so a reentry that fell through here would push and
5298        // open a second pull request. `self.state.merge` is set exactly once
5299        // this node (or `land`) has already produced a verdict, under every
5300        // mode, which is what "already done" actually means here.
5301        if self.state.merge.is_some() {
5302            return Ok(());
5303        }
5304        let Some(winner) = self.state.winner().cloned() else {
5305            return Ok(());
5306        };
5307        let repo = self.state.repo.clone();
5308        let base = self.state.base_branch.clone();
5309        let mode = self.state.config.merge.mode;
5310        let style = self.state.config.merge.style;
5311        let pr = pr_message(&self.state, winner.label);
5312        let message = pr.commit_message();
5313
5314        let outcome = match mode {
5315            MergeMode::None => MergeOutcome {
5316                mode,
5317                ok: true,
5318                detail: manual_merge_command(style, &repo, &winner.branch, &message),
5319            },
5320            MergeMode::Local => {
5321                let on = git::current_branch(&repo).await?;
5322                if on.as_deref() != Some(base.as_str()) {
5323                    MergeOutcome {
5324                        mode,
5325                        ok: false,
5326                        detail: format!(
5327                            "{} has {} checked out, not the base branch {base}",
5328                            repo.display(),
5329                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
5330                        ),
5331                    }
5332                } else if !git::is_clean(&repo).await? {
5333                    MergeOutcome {
5334                        mode,
5335                        ok: false,
5336                        detail: format!("{} is dirty; refusing to merge", repo.display()),
5337                    }
5338                } else {
5339                    let out = match style {
5340                        MergeStyle::Merge => {
5341                            git::merge_no_ff(&repo, &winner.branch, &message).await?
5342                        }
5343                        MergeStyle::Squash => {
5344                            git::merge_squash(&repo, &winner.branch, &message).await?
5345                        }
5346                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5347                    };
5348                    MergeOutcome {
5349                        mode,
5350                        ok: out.ok(),
5351                        detail: if out.ok() { out.stdout } else { out.stderr },
5352                    }
5353                }
5354            }
5355            MergeMode::Pr => {
5356                let remote = self.state.config.merge.remote.clone();
5357                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5358                if !pushed.ok() {
5359                    MergeOutcome {
5360                        mode,
5361                        ok: false,
5362                        detail: pushed.stderr,
5363                    }
5364                } else {
5365                    let out =
5366                        gh_pr_create(&winner.worktree, &base, &winner.branch, &pr.title, &pr.body)
5367                            .await;
5368                    match out {
5369                        Ok(url) => MergeOutcome {
5370                            mode,
5371                            ok: true,
5372                            detail: url,
5373                        },
5374                        Err(e) => MergeOutcome {
5375                            mode,
5376                            ok: false,
5377                            detail: e.to_string(),
5378                        },
5379                    }
5380                }
5381            }
5382        };
5383
5384        self.state.status = match (mode, outcome.ok) {
5385            (MergeMode::None, _) => RunStatus::Ready,
5386            (_, true) => RunStatus::Merged,
5387            (_, false) => RunStatus::Blocked,
5388        };
5389        self.state.event(
5390            "merge",
5391            format!(
5392                "{:?}: {}",
5393                mode,
5394                outcome.detail.lines().next().unwrap_or("")
5395            ),
5396        );
5397        self.state.merge = Some(outcome);
5398        self.state.save()?;
5399
5400        // The PR is open and the run would historically stop here, leaving the
5401        // operator to watch checks, feed review comments back to a fixer, and
5402        // merge. That was done by hand six times in one session before this
5403        // existed. Opt-in, because merging is the one irreversible thing magi
5404        // can do to a repository.
5405        if self.state.config.graph.land
5406            && mode == MergeMode::Pr
5407            && self.state.status == RunStatus::Merged
5408        {
5409            self.run_land().await?;
5410        }
5411        // `run_land` may have left `status` at `Landing` - still waiting on
5412        // CI or the owner's approval, not actually settled - so this has to
5413        // read whatever `status` ended up as here, not the `Merged` this
5414        // function set a few lines up.
5415        self.settle_questions();
5416        Ok(())
5417    }
5418
5419    /// Enter `land`.
5420    ///
5421    /// Shared between a fresh run's first pass through [`Runner::merge`] and
5422    /// a resumed run's re-entry. `land::land` itself is what serialises the
5423    /// two git-mutating moments inside the loop — the rebase push and
5424    /// `gh pr merge` — per repository (see its own doc); nothing here needs
5425    /// to hold a lock across the whole call, and doing so would serialise
5426    /// this run's CI wait against a *different* run's land-approval resume
5427    /// in the same repository, which is exactly the "must not wait on
5428    /// another task" property the daemon's slot-freeing exists to give.
5429    async fn run_land(&mut self) -> Result<()> {
5430        let url = self
5431            .state
5432            .merge
5433            .as_ref()
5434            .map(|m| m.detail.clone())
5435            .unwrap_or_default();
5436        let url = url.lines().next().unwrap_or("").trim().to_owned();
5437        if !url.starts_with("http") {
5438            return Ok(());
5439        }
5440        // A land failure is not a lost run: the work is on a branch and the
5441        // pull request is open, which is exactly where a human takes over.
5442        match land::land(&mut self.state, &url).await {
5443            Ok(pr) if self.state.parked => {
5444                // `land` already saved the parked marker; nothing here
5445                // overrides `status` back to a terminal value while an
5446                // approval is still outstanding.
5447                let _ = pr;
5448            }
5449            Ok(pr) => {
5450                self.state.status = match pr.state {
5451                    land::PrLifecycle::Merged => RunStatus::Merged,
5452                    _ => RunStatus::Blocked,
5453                };
5454                // Downstream of a confirmed merge only - see
5455                // `bump::should_release_bump`'s own doc for why this one
5456                // check covers all three of `land`'s success paths.
5457                // Best-effort: the run already landed, so a failure here
5458                // (the decision call, `gh`, `cargo`) is recorded and never
5459                // turns a landed run into a failed one.
5460                if bump::should_release_bump(self.state.status)
5461                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
5462                {
5463                    // Deliberately only an event: most `Err`s here mean the
5464                    // bump did not apply (no `Cargo.toml`, no agent
5465                    // installed, an unusable decision), not that a release
5466                    // PR is stranded. `after_merge` raises its own notice
5467                    // once a PR exists and needs a human.
5468                    self.state
5469                        .event("bump", format!("release bump skipped: {e:#}"));
5470                }
5471                self.state.save()?;
5472            }
5473            Err(e) => {
5474                self.state.status = RunStatus::Blocked;
5475                self.state.event("land", format!("gave up: {e}"));
5476                self.state.save()?;
5477            }
5478        }
5479        Ok(())
5480    }
5481
5482    // -------------------------------------------------------------- helpers
5483
5484    /// Fetch or create a seat, keeping its conversation across nodes.
5485    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
5486        if let Some(existing) = self.state.seats.get(key)
5487            && existing.agent == agent
5488        {
5489            return existing.clone();
5490        }
5491        let fresh = SeatState::new(key, agent, self.state.seed);
5492        self.state.seats.insert(key.to_owned(), fresh.clone());
5493        fresh
5494    }
5495
5496    /// A candidate rendered for judging, with the leak policy applied.
5497    fn view(&self, c: &Candidate) -> CandidateView {
5498        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
5499            .unwrap_or_default();
5500        let (patch, _) = blind::sanitize_patch(
5501            &format!("candidate {} patch", c.label),
5502            &raw,
5503            &self.state.config.blind,
5504        );
5505        CandidateView {
5506            label: c.label,
5507            branch: c.branch.clone(),
5508            summary: c.summary.clone(),
5509            stat: c.stat.clone(),
5510            patch,
5511        }
5512    }
5513
5514    /// The full candidate set as prompt text, for seats with no live session.
5515    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
5516        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
5517        prompt::judge(
5518            "(see above)",
5519            &views,
5520            self.roles.judges.len(),
5521            base_short,
5522            "en",
5523        )
5524    }
5525
5526    /// Anonymised transcript for judge `self_idx`.
5527    ///
5528    /// The initial rankings are always the opening statements. Seeding them
5529    /// only when no turn had been taken yet meant every judge after the first
5530    /// argued against a single voice instead of against the actual split — the
5531    /// disagreement is the information, so it is always on the table.
5532    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
5533        let mut turns = Vec::new();
5534        for j in &self.state.judgements {
5535            if j.ranking.is_empty() {
5536                continue;
5537            }
5538            let reasons = j
5539                .reasons
5540                .iter()
5541                .map(|(k, v)| format!("- {k}: {v}"))
5542                .collect::<Vec<_>>()
5543                .join("\n");
5544            turns.push(Turn {
5545                who: format!("Judge {} (opening ranking)", j.judge),
5546                is_self: j.judge == self_idx + 1,
5547                body: format!(
5548                    "Ranked {}{}{reasons}",
5549                    j.ranking.iter().collect::<String>(),
5550                    if reasons.is_empty() {
5551                        ""
5552                    } else {
5553                        ", because:\n"
5554                    }
5555                ),
5556            });
5557        }
5558        for t in self
5559            .state
5560            .deliberation
5561            .iter()
5562            .flat_map(|r| r.turns.iter())
5563            .chain(current)
5564        {
5565            turns.push(Turn {
5566                who: format!("Judge {}", t.judge),
5567                is_self: t.judge == self_idx + 1,
5568                body: t.body.clone(),
5569            });
5570        }
5571        turns
5572    }
5573}
5574
5575/// Does this seat still hold the context a follow-up prompt would rely on?
5576fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
5577    agent::has_session(spec.kind, seat, sessions)
5578}
5579
5580/// The next entry in `roster` after `start`, never wrapping back to the
5581/// front, whose id is not in `tried` yet.
5582///
5583/// Starts one past `start` rather than at the front of `roster`: `start` is
5584/// the seat's own original position, and a seat whose candidate slot already
5585/// sits on the roster's second entry must fall through to the third next, not
5586/// restart at the first — which is very likely a different candidate's own
5587/// agent already. Never wraps back past `start`, for the same reason: an
5588/// entry earlier in the roster than the seat's own position is almost
5589/// certainly some *other* candidate slot's own agent, and once the tail of
5590/// the roster is exhausted there are no more untried agents for *this* seat
5591/// to fall through to — the caller's fallback chain ends there, exactly as
5592/// "no further untried agents remain in the list for that seat" asks for.
5593///
5594/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
5595/// the same id twice (an operator's `roles.implementers` typo, or a
5596/// `[[agents]]` list reused across roles) must not let
5597/// [`Runner::resume_quota_losses`] retry that id forever — one forward pass
5598/// over `roster` either finds an untried id or runs out, so this always
5599/// terminates regardless of duplicates.
5600fn next_untried_implementer<'a>(
5601    roster: &'a [AgentSpec],
5602    start: usize,
5603    tried: &BTreeSet<String>,
5604) -> Option<&'a AgentSpec> {
5605    roster
5606        .get(start + 1..)?
5607        .iter()
5608        .find(|s| !tried.contains(&s.id))
5609}
5610
5611/// Did this reply report running a command whose own CLI never confirmed an
5612/// exit status?
5613///
5614/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
5615/// command *finished* (see that type's own doc), so this can only be `true`
5616/// for a command whose completion event carried no readable exit code — not
5617/// for one that simply is not mentioned at all. That is the one signal this
5618/// crate can read, from the same record `state.jobs` renders, about a reply
5619/// standing next to work its own CLI cannot vouch for finishing; it is
5620/// deliberately not a check on the exit code's *value* (a fixer legitimately
5621/// runs a command that fails mid-iteration before it succeeds) and not a
5622/// guess at a command still running in the background (which emits no event
5623/// at all, and so leaves no evidence here to find).
5624fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
5625    commands.iter().any(|c| c.exit_code.is_none())
5626}
5627
5628/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
5629/// trusted as a verified no-op — the adoption guard's own text-level half.
5630///
5631/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
5632/// timed out): a marker only earns the benefit of the doubt from a turn the
5633/// CLI itself vouches for finishing properly, the same house style
5634/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
5635/// *fix* report to for `commands`. A candidate that timed out, exited
5636/// non-zero, or left a command unconfirmed is read as the ordinary loss it
5637/// is, whatever prose it wrote — this returns `None` before it ever looks at
5638/// `text`. The remaining guards (the tree really is empty, the evidence is
5639/// non-empty) are the caller's: this only reads what the reply *claimed*.
5640fn verified_noop_claim(
5641    usable: bool,
5642    commands: &[agent::CommandEvidence],
5643    text: &str,
5644) -> Option<String> {
5645    (usable && !has_unconfirmed_command(commands))
5646        .then(|| verdict::verified_noop(text))
5647        .flatten()
5648}
5649
5650fn short(commit: &str) -> String {
5651    commit.chars().take(7).collect()
5652}
5653
5654fn make_executable(path: &Path) -> Result<()> {
5655    #[cfg(unix)]
5656    {
5657        use std::os::unix::fs::PermissionsExt as _;
5658        let mut perms = std::fs::metadata(path)?.permissions();
5659        perms.set_mode(0o755);
5660        std::fs::set_permissions(path, perms)?;
5661    }
5662    #[cfg(not(unix))]
5663    {
5664        let _ = path;
5665    }
5666    Ok(())
5667}
5668
5669/// What every seat in one batch shares: where the answers are attributed, the
5670/// prompt overlay they inherit, and the build cache they are told to use.
5671///
5672/// A struct rather than four more parameters: `wave` also needs the run's
5673/// state (to record who is answering right now) and the attempt number, and
5674/// eight positional arguments is both unreadable and a clippy error.
5675struct WaveCtx<'a> {
5676    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
5677    /// paid for it.
5678    run: &'a str,
5679    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
5680    node: &'a str,
5681    prompts: &'a Prompts,
5682    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
5683    cache: Option<&'a Path>,
5684    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
5685    /// `JobRecord::round`. `None` for every other node.
5686    round: Option<usize>,
5687}
5688
5689/// Run one job, honouring the parallelism budget.
5690async fn run_one(
5691    job: SeatJob,
5692    sem: Arc<Semaphore>,
5693    ctx: &WaveCtx<'_>,
5694    state: &mut RunState,
5695    attempt: usize,
5696) -> (SeatState, AgentOutcome) {
5697    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
5698        .await
5699        .pop()
5700        .expect("one job in, one result out");
5701    (seat, out)
5702}
5703
5704/// Run every job concurrently, capped by the semaphore, preserving order.
5705///
5706/// Every seat in the batch is recorded into [`RunState::active`] before the
5707/// wave starts and cleared as each answer lands, so the run's own record says
5708/// who is still being waited on rather than only who finished.
5709async fn wave(
5710    jobs: Vec<SeatJob>,
5711    sem: Arc<Semaphore>,
5712    ctx: &WaveCtx<'_>,
5713    state: &mut RunState,
5714    attempt: usize,
5715) -> Vec<(usize, SeatState, AgentOutcome)> {
5716    let WaveCtx {
5717        run,
5718        node,
5719        prompts,
5720        cache,
5721        round,
5722    } = *ctx;
5723    for job in &jobs {
5724        state.seat_started(node, &job.seat.key, job.timeout, attempt);
5725    }
5726    if let Err(e) = state.save() {
5727        // A failed persist of "who is answering right now" must not abort the
5728        // wave: the seats are already being asked, and the alternative is
5729        // losing the answers to save a status line nobody may even be
5730        // watching.
5731        tracing::warn!("could not persist in-progress seats: {e:#}");
5732    }
5733    // Hold the shared build cache's lease for the whole batch, not per job:
5734    // several candidates (an implement wave) or a fixer legitimately share
5735    // one cache concurrently within this run, and that stays untouched — a
5736    // single lease taken once for the whole wave and released once it is
5737    // done is what stops a *different* borrower (another run's own wave, its
5738    // e2e/gate, a human's `magi review`) from interleaving a build into the
5739    // same directory while this one is in flight. Best-effort, not
5740    // all-or-nothing: a wave that cannot get the lease within its own
5741    // longest job's budget still runs — an hour of paid implementer calls is
5742    // not thrown away over cache contention — but every write-allowed seat
5743    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
5744    // below), the same fallback a read-only seat always gets, rather than
5745    // building into a directory this run was never granted. The identity
5746    // record is still invalidated below either way, so the next tracked
5747    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
5748    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
5749    let wait_started = Instant::now();
5750    let cache_guard = if let Some(cache_dir) = cache {
5751        if jobs_had_a_writer {
5752            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
5753            let budget = jobs
5754                .iter()
5755                .map(|j| j.timeout)
5756                .max()
5757                .unwrap_or(Duration::from_secs(60));
5758            acquire_cache_lease(state, cache_dir, &owner, budget, node)
5759                .await
5760                .ok()
5761        } else {
5762            None
5763        }
5764    } else {
5765        None
5766    };
5767    // Carved out of each job's own budget, not added on top of it: a seat
5768    // that waited behind the lease must not also get its full timeout
5769    // afterward, or a run contended on the cache could double the time it
5770    // spends per wave. `saturating_sub` floors at zero rather than
5771    // wrapping - a job whose whole budget was spent waiting starts with
5772    // none left, which is the honest number, not a free minimum.
5773    let waited_for_lease = wait_started.elapsed();
5774    let mut set = tokio::task::JoinSet::new();
5775    let overlay = prompts.overlay(node);
5776    for (i, mut job) in jobs.into_iter().enumerate() {
5777        job.timeout = job.timeout.saturating_sub(waited_for_lease);
5778        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
5779        if cache.is_some() {
5780            job.prompt.push('\n');
5781            job.prompt
5782                .push_str(&prompt::build_cache_note(node, job.allow_write));
5783        }
5784        let sem = Arc::clone(&sem);
5785        let run = run.to_owned();
5786        let node = node.to_owned();
5787        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
5788        // `prompt::build_cache_note`'s doc for why setting it anyway is
5789        // exactly how a sandboxed reviewer's write refusal got reported as a
5790        // defect in the patch, not a property of its own seat. And a
5791        // write-allowed one is handed it only when the lease above was
5792        // actually acquired: a wave that could not get it (`cache_guard` is
5793        // `None`, see its own comment) must not send seats to build into a
5794        // directory this run does not hold - that is the exact concurrent,
5795        // unmanaged-write race this module exists to prevent, not something
5796        // "proceeding anyway" is allowed to reintroduce.
5797        let cache = cache
5798            .filter(|_| job.allow_write && cache_guard.is_some())
5799            .map(Path::to_path_buf);
5800        set.spawn(async move {
5801            let _permit = sem.acquire().await;
5802            let mut seat = job.seat;
5803            let out = agent::invoke(
5804                &job.spec,
5805                &mut seat,
5806                &Invocation {
5807                    cwd: &job.cwd,
5808                    prompt: &job.prompt,
5809                    timeout: job.timeout,
5810                    allow_write: job.allow_write,
5811                    sessions: job.sessions,
5812                    artifacts: &job.artifacts,
5813                    stem: &job.stem,
5814                    run: &run,
5815                    node: &node,
5816                    cache_dir: cache.as_deref(),
5817                    attachments: &[],
5818                },
5819            )
5820            .await;
5821            let out = match out {
5822                Ok(o) if o.usable() => AgentOutcome::Ok(o),
5823                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
5824                // Billed work the CLI failed to hand over is not an ordinary
5825                // failure, but its text is the CLI's raw error JSON, not an
5826                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
5827                // read it as one by forgetting to check. `usable()` is always
5828                // false here (dropped implies an empty response), so this has
5829                // to be checked before the catch-all `Failed` below or the
5830                // one shape this exists for is lost with the rest.
5831                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
5832                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
5833                Ok(o) => AgentOutcome::Failed(format!(
5834                    "exited with {:?} and no usable output",
5835                    o.exit_code
5836                )),
5837                Err(e) => AgentOutcome::Failed(e.to_string()),
5838            };
5839            (i, seat, out)
5840        });
5841    }
5842    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
5843    while let Some(joined) = set.join_next().await {
5844        let (i, seat, out) = match joined {
5845            Ok(v) => v,
5846            // No seat to clear: a panicked task never reported which one it
5847            // was. The defensive sweep below this loop is what stops that
5848            // seat's `active` entry from surviving forever.
5849            Err(e) => {
5850                tracing::error!("agent task panicked: {e}");
5851                continue;
5852            }
5853        };
5854        state.seat_finished(&seat.key);
5855        record_jobs(state, node, round, &seat.key, &out);
5856        if let Err(e) = state.save() {
5857            tracing::warn!("could not persist a seat's completion: {e:#}");
5858        }
5859        if collected.len() <= i {
5860            collected.resize_with(i + 1, || None);
5861        }
5862        collected[i] = Some((i, seat, out));
5863    }
5864    // Belt-and-braces for the panic branch above: every seat this exact batch
5865    // started shares this `(node, attempt)` pair, and every seat that finished
5866    // normally already cleared itself, so anything left tagged with it here
5867    // can only be a panicked task's leftover. Cleared unconditionally rather
5868    // than left to read as still answering forever.
5869    if state
5870        .active
5871        .values()
5872        .any(|a| a.node == node && a.attempt == attempt)
5873    {
5874        state
5875            .active
5876            .retain(|_, a| !(a.node == node && a.attempt == attempt));
5877        if let Err(e) = state.save() {
5878            tracing::warn!("could not persist the end of a wave: {e:#}");
5879        }
5880    }
5881    // Whether or not the lease above was actually held, several worktrees
5882    // may just have built into the cache with nothing here able to name one
5883    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
5884    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
5885    // clean it might not have strictly needed; trusting a stale match would
5886    // cost it a wrong answer.
5887    if let Some(cache_dir) = cache
5888        && jobs_had_a_writer
5889    {
5890        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
5891    }
5892    if let Some(guard) = cache_guard {
5893        guard.release();
5894    }
5895    collected.into_iter().flatten().collect()
5896}
5897
5898/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
5899/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
5900/// this is data collection, not the fix-specific completion contract in
5901/// [`Runner::continue_fix_report`], and applies regardless of which node
5902/// asked.
5903///
5904/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
5905/// evidence from; `Failed` does not, and correctly contributes nothing — a
5906/// timeout or crash is not itself evidence about a command the seat may have
5907/// started.
5908fn record_jobs(
5909    state: &mut RunState,
5910    node: &str,
5911    round: Option<usize>,
5912    seat: &str,
5913    out: &AgentOutcome,
5914) {
5915    let commands: &[agent::CommandEvidence] = match out {
5916        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
5917        AgentOutcome::Failed(_) => &[],
5918    };
5919    let checked_at = Timestamp::now();
5920    for c in commands {
5921        state.jobs.push(JobRecord {
5922            node: node.to_owned(),
5923            round,
5924            seat: seat.to_owned(),
5925            id: c.id.clone(),
5926            description: c.description.clone(),
5927            checked_at,
5928            status: match c.exit_code {
5929                Some(0) => JobStatus::Completed,
5930                Some(_) => JobStatus::Failed,
5931                None => JobStatus::Unknown,
5932            },
5933            exit_code: c.exit_code,
5934            result_summary: c.result_summary.clone(),
5935            source: c.source.clone(),
5936        });
5937    }
5938}
5939
5940/// Is a review round clean, given how many reviewer seats answered against
5941/// how many the round expected?
5942///
5943/// A seat that never answered (timeout, crash, unparsable output) is not a
5944/// seat that read the patch and found nothing — treating it as such is
5945/// exactly the bug this function exists to close. Under the default `block`
5946/// policy a missing seat can never be clean; `warn` still requires the seats
5947/// that *did* answer to have found nothing blocking and verification to be
5948/// green.
5949///
5950/// `quota_missing` narrows that `block` default for exactly one cause of
5951/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
5952/// a session limit lifts by the very next round buys nothing — the seat is
5953/// asked again with the same quota — so once every missing seat is accounted
5954/// for by a quota loss (and at least one seat *did* answer, so a decision has
5955/// something to rest on) the round is decided on the panel that could answer,
5956/// same as `warn` would. A panel that lost every seat to quota is not
5957/// decided here: `answered == 0` falls through to the existing `block`
5958/// fallback so a fully collapsed panel still waits rather than landing on no
5959/// review at all.
5960fn round_is_clean(
5961    blocking: usize,
5962    e2e_ok: bool,
5963    answered: usize,
5964    expected: usize,
5965    quota_missing: usize,
5966    policy: IncompleteReviewPolicy,
5967) -> bool {
5968    if blocking != 0 || !e2e_ok {
5969        return false;
5970    }
5971    if answered == expected || policy == IncompleteReviewPolicy::Warn {
5972        return true;
5973    }
5974    answered > 0 && expected - answered <= quota_missing
5975}
5976
5977/// The review loop's own conclusion, derived entirely from its persisted
5978/// round records and the round budget that produced them — never from
5979/// `status`, so a reentry (or `gate`/`merge` reading it independently)
5980/// recomputes the identical answer regardless of what an earlier node in the
5981/// same walk, or a previous walk, did to `status`.
5982///
5983/// `None` while more rounds remain to try, including when review never ran
5984/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
5985/// gone clean, or the budget is spent, or the tree has stopped moving (see
5986/// [`STAGNANT_LIMIT`]), the answer is one of two things:
5987///
5988/// - An incomplete panel that raised nothing is missing input, not a
5989///   verified tree — never a hand-off candidate, whatever verification said
5990///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
5991/// - Otherwise, green e2e on the last round hands off (see
5992///   [`Runner::stop_reviewing`]); red e2e blocks.
5993///
5994/// A last round whose own verification is still `ResourceBlocked` — magi
5995/// itself never got a command to run, not evidence the patch is broken —
5996/// is neither: this returns `None` for it too, the same as "more rounds
5997/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
5998/// handling of that shape) instead of this cheap recomputation guessing a
5999/// verdict a real attempt never produced.
6000fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
6001    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
6002        return Some(RunStatus::Gating);
6003    }
6004    let last = reviews.last()?;
6005    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
6006    if reviews.len() < max_rounds && !stagnant {
6007        return None;
6008    }
6009    if last.incomplete() && last.blocking == 0 {
6010        return Some(RunStatus::Blocked);
6011    }
6012    if last.e2e_status() == E2eStatus::ResourceBlocked {
6013        return None;
6014    }
6015    Some(if last.e2e.iter().all(CommandOutcome::ok) {
6016        RunStatus::Gating
6017    } else {
6018        RunStatus::Blocked
6019    })
6020}
6021
6022/// How long a re-ask may take, given the budget the first attempt had.
6023///
6024/// A `nudged` retry is a request to restate an answer the seat has already
6025/// worked out: it carries no new work, so it does not deserve the original
6026/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
6027/// 133 seconds while a third sat for over ten minutes on a resumed session
6028/// holding 410 KB of prior output - and because the retry had inherited the
6029/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
6030/// judging round whose other seats were long finished.
6031///
6032/// A quarter of the budget, with a floor so that a deliberately short timeout
6033/// does not collapse to nothing. A retry that re-sends the whole prompt
6034/// (because the seat kept no context) is the original job again, and keeps the
6035/// original budget.
6036fn retry_budget(full: Duration, nudged: bool) -> Duration {
6037    if nudged {
6038        (full / 4).max(Duration::from_secs(120)).min(full)
6039    } else {
6040        full
6041    }
6042}
6043
6044/// Run a wave and parse each reply, re-asking the seats whose reply was
6045/// unusable.
6046///
6047/// The re-ask is a nudge rather than the whole prompt again when the seat still
6048/// holds its conversation, which is the difference between a cheap retry and
6049/// paying for the entire candidate set twice.
6050///
6051/// A seat that hits a rate limit is **not** re-asked: the same call will fail
6052/// the same way until the limit resets, so spending a retry attempt on it is
6053/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
6054/// like any other absent seat — the caller decides whether the panel still has
6055/// a quorum.
6056#[allow(clippy::too_many_arguments)]
6057async fn ask_json_wave<T>(
6058    jobs: Vec<SeatJob>,
6059    sem: Arc<Semaphore>,
6060    retries: usize,
6061    ctx: &WaveCtx<'_>,
6062    losses: &mut Vec<QuotaLoss>,
6063    state: &mut RunState,
6064    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
6065) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6066where
6067    T: serde::de::DeserializeOwned + Send + 'static,
6068{
6069    let n = jobs.len();
6070    let originals: Vec<SeatJob> = jobs;
6071    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
6072    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
6073    // Which attempt each seat's `done[i]` reflects — 0 for a first-ask
6074    // answer, N once it has gone through N nudges. Read back once this
6075    // returns, so a caller building a history record (`ReviewRecord`) can
6076    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
6077    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
6078    // field's own doc.
6079    let mut attempts_used: Vec<usize> = vec![0; n];
6080    let mut pending: Vec<usize> = (0..n).collect();
6081
6082    for attempt in 0..=retries {
6083        if pending.is_empty() {
6084            break;
6085        }
6086        let mut batch = Vec::with_capacity(pending.len());
6087        for &i in &pending {
6088            let src = &originals[i];
6089            // The prompt and the budget are one decision: a nudge restates
6090            // finished work, a re-sent prompt redoes it.
6091            let (prompt, timeout) = if attempt == 0 {
6092                (src.prompt.clone(), src.timeout)
6093            } else {
6094                let why = done[i]
6095                    .as_ref()
6096                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
6097                    .unwrap_or_else(|| "no parsable answer".to_owned());
6098                let nudge = prompt::nudge(&why);
6099                let nudged = has_context(&src.spec, &seats[i], src.sessions);
6100                let prompt = if nudged {
6101                    nudge
6102                } else {
6103                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
6104                };
6105                (prompt, retry_budget(src.timeout, nudged))
6106            };
6107            batch.push(SeatJob {
6108                spec: src.spec.clone(),
6109                seat: seats[i].clone(),
6110                cwd: src.cwd.clone(),
6111                prompt,
6112                timeout,
6113                allow_write: src.allow_write,
6114                sessions: src.sessions,
6115                artifacts: src.artifacts.clone(),
6116                stem: if attempt == 0 {
6117                    src.stem.clone()
6118                } else {
6119                    format!("{}-retry{attempt}", src.stem)
6120                },
6121            });
6122        }
6123
6124        if attempt > 0 {
6125            let seats_out: Vec<&str> = pending
6126                .iter()
6127                .map(|&i| originals[i].seat.key.as_str())
6128                .collect();
6129            state.event(
6130                ctx.node,
6131                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
6132            );
6133        }
6134        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
6135        let mut still = Vec::new();
6136        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
6137            seats[i] = seat;
6138            let (parsed, quota) = match out {
6139                AgentOutcome::Ok(o) => (
6140                    match verdict::extract_json::<T>(&o.text) {
6141                        Ok(v) => match validate(&v) {
6142                            Ok(()) => Ok((v, o)),
6143                            Err(e) => Err(e),
6144                        },
6145                        Err(e) => Err(e),
6146                    },
6147                    false,
6148                ),
6149                AgentOutcome::Quota(o) => {
6150                    losses.push(QuotaLoss {
6151                        seat: originals[i].seat.key.clone(),
6152                        node: ctx.node.to_owned(),
6153                        at: Timestamp::now(),
6154                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6155                    });
6156                    (
6157                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
6158                        true,
6159                    )
6160                }
6161                // Not a parseable answer, but also not worth a special-cased
6162                // retry here: the nudge loop above already re-asks anything
6163                // that fails to parse, which is exactly what a dropped stream
6164                // needs. Just don't hand its raw error JSON to `extract_json`.
6165                AgentOutcome::Dropped(o) => {
6166                    let why = o
6167                        .dropped
6168                        .as_ref()
6169                        .map(|d| d.why.as_str())
6170                        .unwrap_or("the CLI ended the stream without delivering its answer");
6171                    (
6172                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
6173                        false,
6174                    )
6175                }
6176                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
6177            };
6178            let failed = parsed.is_err();
6179            done[i] = Some(parsed);
6180            attempts_used[i] = attempt;
6181            // Do not re-ask a rate-limited seat (quota) — a retry is known to
6182            // fail the same way; and never re-ask a seat that already parsed.
6183            if failed && !quota {
6184                still.push(i);
6185            }
6186        }
6187        pending = still;
6188    }
6189
6190    seats
6191        .into_iter()
6192        .zip(done)
6193        .zip(attempts_used)
6194        .map(|((seat, res), attempts)| {
6195            (
6196                seat,
6197                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
6198                attempts,
6199            )
6200        })
6201        .collect()
6202}
6203
6204/// Acquire the shared build cache's lease, waiting out contention within
6205/// `budget` (never past it — see AGENTS.md's build-cache section on why an
6206/// unbounded wait is never acceptable).
6207///
6208/// A first, non-blocking check happens before ever waiting; if it finds the
6209/// lease busy, that fact is logged as a `verify` event *and* flushed with
6210/// [`RunState::save`] immediately — not only once the wait finally succeeds
6211/// or gives up — so a `magi show` run by a different process while this one
6212/// is still waiting reads a `run.json` that says so, rather than whatever it
6213/// looked like before the wait started. The same applies to the terminal
6214/// failure: logged and saved before this returns `Err`, so a caller that
6215/// could not get the lease at all still leaves a legible record of why.
6216async fn acquire_cache_lease(
6217    state: &mut RunState,
6218    cache_dir: &Path,
6219    owner: &crate::cache::Owner,
6220    budget: Duration,
6221    context: &str,
6222) -> Result<crate::cache::Guard> {
6223    let home = crate::run::home();
6224    let started = Instant::now();
6225    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
6226        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
6227        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
6228        Err(e) => {
6229            state.event(
6230                "verify",
6231                format!("{context}: could not check the shared build cache: {e:#}"),
6232            );
6233            if let Err(e2) = state.save() {
6234                tracing::warn!("could not persist a cache-check failure: {e2:#}");
6235            }
6236            return Err(e);
6237        }
6238    };
6239    state.event(
6240        "verify",
6241        format!(
6242            "{context}: waiting for the shared build cache at {} ({})",
6243            cache_dir.display(),
6244            busy.describe()
6245        ),
6246    );
6247    if let Err(e) = state.save() {
6248        tracing::warn!("could not persist a cache wait: {e:#}");
6249    }
6250    let remaining = budget.saturating_sub(started.elapsed());
6251    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
6252        Ok(g) => Ok(g),
6253        Err(e) => {
6254            state.event("verify", format!("{context}: {e:#}"));
6255            if let Err(e2) = state.save() {
6256                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
6257            }
6258            Err(e)
6259        }
6260    }
6261}
6262
6263/// Run `body` — a verify command batch — while holding the shared build
6264/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
6265/// never interleave with another borrower's build against the same
6266/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
6267/// timeout, or a human's own `magi review`. See the `cache` module doc for
6268/// why this matters more than Cargo's own per-target locking covers — two
6269/// *different* worktrees building the same package name/version into one
6270/// cache directory is a staleness bug, not a lock contention one.
6271///
6272/// The wait for the lease is carved out of `budget`, never on top of it —
6273/// `body` is handed whatever is left, so a caller's own node timeout is the
6274/// only clock involved, exactly what AGENTS.md's build-cache section asks
6275/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
6276/// cache configured at all — this is a pass-through: `body` runs with the
6277/// full budget and nothing is leased.
6278///
6279/// A lease that cannot be acquired within `budget` is reported as a single
6280/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
6281/// verification — the same shape a spawn failure already takes in
6282/// [`run_commands`], so a caller need not special-case it.
6283#[allow(clippy::too_many_arguments)]
6284async fn with_cache_lease<'s, F, Fut>(
6285    state: &'s mut RunState,
6286    cache_dir: Option<&Path>,
6287    node: &str,
6288    seat: &str,
6289    worktree: &Path,
6290    head: &str,
6291    budget: Duration,
6292    context: &str,
6293    body: F,
6294) -> (Vec<CommandOutcome>, bool)
6295where
6296    F: FnOnce(&'s mut RunState, Duration) -> Fut,
6297    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
6298{
6299    let Some(cache_dir) = cache_dir else {
6300        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
6301        return (outcomes, retried);
6302    };
6303    let home = crate::run::home();
6304    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
6305    let started = Instant::now();
6306    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
6307        Ok(g) => g,
6308        Err(e) => {
6309            return (
6310                vec![CommandOutcome {
6311                    command: "(waiting for the shared build cache)".to_owned(),
6312                    code: None,
6313                    output_tail: e.to_string(),
6314                    duration_ms: started.elapsed().as_millis() as u64,
6315                    resource_blocked: true,
6316                }],
6317                false,
6318            );
6319        }
6320    };
6321    let identity = crate::cache::Identity::new(worktree, head);
6322    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
6323        // A failed freshness check means this process cannot vouch for what
6324        // is sitting in the cache right now - on Windows this is exactly the
6325        // "a stale test executable is still locked, `cargo clean -p` cannot
6326        // remove it" case the evidence log records. Running verify anyway
6327        // and reporting whatever it says would let a result nobody can trust
6328        // stand for the tree it claims to have checked; fail the step
6329        // instead of the patch.
6330        state.event(
6331            "verify",
6332            format!(
6333                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
6334                worktree.display(),
6335                short(head)
6336            ),
6337        );
6338        guard.release();
6339        return (
6340            vec![CommandOutcome {
6341                command: "(confirming the shared build cache is fresh)".to_owned(),
6342                code: None,
6343                output_tail: e.to_string(),
6344                duration_ms: started.elapsed().as_millis() as u64,
6345                resource_blocked: true,
6346            }],
6347            false,
6348        );
6349    }
6350    let remaining = budget.saturating_sub(started.elapsed());
6351    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
6352    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
6353    // `start_kill`); confirm it actually has before handing the directory to
6354    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
6355    // for what this can and cannot see.
6356    if !timed_out_pids.is_empty() {
6357        wait_for_timed_out_children_to_die(&timed_out_pids).await;
6358    }
6359    guard.release();
6360    (outcomes, retried)
6361}
6362
6363/// Poll `pids` — commands [`run_commands`] reports as still running when its
6364/// own timeout elapsed — until every one is confirmed gone, or
6365/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
6366///
6367/// Real confirmation where confirmation is possible, not a substitute for
6368/// full process-tree observation: a grandchild the timed-out process spawned
6369/// and that survives independently of it is invisible to a pid check the
6370/// same way it always was, and continuing to observe and collect *that*
6371/// stays a different piece of work with its own owner. This only narrows a
6372/// fixed blind wait into an actual check of the pids this process does know
6373/// about.
6374async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
6375    wait_for_pids_with(
6376        pids,
6377        crate::proc::pid_alive,
6378        LEASE_RELEASE_POLL,
6379        LEASE_RELEASE_MAX_WAIT,
6380    )
6381    .await;
6382}
6383
6384/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
6385/// interval and ceiling supplied by the caller, so the polling *logic* -
6386/// returns as soon as every pid reports dead, gives up at the ceiling
6387/// otherwise - is testable on millisecond durations without asking the real
6388/// OS about a pid at all.
6389async fn wait_for_pids_with<F: Fn(u32) -> bool>(
6390    pids: &[u32],
6391    alive: F,
6392    poll: Duration,
6393    max_wait: Duration,
6394) {
6395    let deadline = Instant::now() + max_wait;
6396    loop {
6397        if pids.iter().all(|&pid| !alive(pid)) {
6398            return;
6399        }
6400        if Instant::now() >= deadline {
6401            return;
6402        }
6403        tokio::time::sleep(poll).await;
6404    }
6405}
6406
6407/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
6408/// admission that it could not even get a verify command to run, as opposed
6409/// to evidence the command actually produced? A caller that would otherwise
6410/// read a resource-blocked outcome as a red command must check this first:
6411/// see [`Runner::gate`], which retries rather than records `Blocked` when
6412/// this is true.
6413fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
6414    outcomes.iter().any(|o| o.resource_blocked)
6415}
6416
6417/// What [`Runner::gate_fix_round`] decided.
6418enum GateFix {
6419    /// The tree changed and `verify.e2e` is still green: run the gate again.
6420    Retry,
6421    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
6422    /// last failure stands and the run ends blocked.
6423    Stop,
6424    /// `verify.e2e` could not run after the fix (magi's own contention):
6425    /// decide nothing now, a later reentry retries.
6426    Defer,
6427}
6428
6429/// Is every red command in `outcomes` an ordinary failure the code could
6430/// explain: it ran, exited non-zero, and said something?
6431///
6432/// A timeout, a spawn failure and a killed process all leave `code` `None`;
6433/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
6434/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
6435/// what the command is stays the gate's business.
6436fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
6437    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
6438    red.peek().is_some()
6439        && red.all(|o| {
6440            !o.resource_blocked
6441                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
6442                && !o.output_tail.trim().is_empty()
6443        })
6444}
6445
6446/// Describe one verify command's outcome for the event log, distinguishing a
6447/// build/link failure — the toolchain never produced a binary to run — from
6448/// an actual test failure, since only the latter is a verdict on the patch.
6449fn e2e_outcome_label(o: &CommandOutcome) -> String {
6450    if o.ok() {
6451        return "pass".to_owned();
6452    }
6453    let reason = if o.build_failed() {
6454        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
6455    } else {
6456        format!("FAIL ({:?})", o.code)
6457    };
6458    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
6459}
6460
6461/// Run `verify.e2e`, retrying once if the first attempt could not build or
6462/// link — a build/link failure is frequently a race against a shared
6463/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
6464/// `verify` event per command, tagged with `context` (normally `"round N"`)
6465/// so the two call sites that need this — the ordinary per-round leg in
6466/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
6467/// it will ever call a round green — read identically in the event log.
6468async fn run_e2e_with_retry(
6469    state: &mut RunState,
6470    shell: &[String],
6471    commands: &[String],
6472    worktree: &Path,
6473    timeout: Duration,
6474    context: &str,
6475) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
6476    let (mut e2e, mut timed_out_pids) = run_commands(
6477        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
6478    )
6479    .await;
6480    for o in &e2e {
6481        state.event(
6482            "verify",
6483            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
6484        );
6485    }
6486    // A build/link failure is not a verdict on the patch — it is frequently a
6487    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
6488    // one retry before letting a red like that decide the round.
6489    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
6490    if verify_retried {
6491        state.event(
6492            "verify",
6493            format!(
6494                "{context}: verify could not build/link, not a test result — retrying once \
6495                 before concluding"
6496            ),
6497        );
6498        let retried = run_commands(
6499            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
6500        )
6501        .await;
6502        e2e = retried.0;
6503        // Both attempts' timeouts matter, not just the last one: the first
6504        // attempt's descendants may still be alive alongside the retry's.
6505        timed_out_pids.extend(retried.1);
6506        for o in &e2e {
6507            state.event(
6508                "verify",
6509                format!(
6510                    "{context}: retry `{}` -> {}",
6511                    o.command,
6512                    e2e_outcome_label(o)
6513                ),
6514            );
6515        }
6516    }
6517    (e2e, verify_retried, timed_out_pids)
6518}
6519
6520/// Run configured shell commands in `cwd`, in order. The second element is
6521/// the pid of every command that hit `timeout` and was still running when
6522/// this stopped waiting on it (best-effort: `None` when the platform did not
6523/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
6524/// that releases a shared resource afterward needs to know.
6525///
6526/// Records `task` into [`RunState::active`] at every command boundary
6527/// (`RunState::task_command`) and clears it once the whole list has run
6528/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
6529/// for minutes with no seat and no output of its own to show for it (see
6530/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
6531/// apart from "not yet run" without this), and this is the only place that
6532/// knows which command is running right now and how many are left. Three
6533/// saves per command — start, not per second — matching the same "only at a
6534/// boundary" rule [`wave`] already follows for seats.
6535#[allow(clippy::too_many_arguments)]
6536async fn run_commands(
6537    state: &mut RunState,
6538    node: &str,
6539    task: &str,
6540    attempt: usize,
6541    shell: &[String],
6542    commands: &[String],
6543    cwd: &Path,
6544    timeout: Duration,
6545) -> (Vec<CommandOutcome>, Vec<u32>) {
6546    if commands.is_empty() {
6547        // Nothing to mark as running and nothing to clear — an empty list
6548        // means "not configured", and touching `active` (or the disk) over
6549        // that would be a write for every round of a repo with no
6550        // `verify.e2e` / `verify.gate` commands at all.
6551        return (Vec::new(), Vec::new());
6552    }
6553    let mut out = Vec::new();
6554    let mut timed_out_pids = Vec::new();
6555    let total = commands.len();
6556    for (idx, command) in commands.iter().enumerate() {
6557        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
6558        if let Err(e) = state.save() {
6559            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
6560        }
6561        let started = Instant::now();
6562        let mut cmd = tokio::process::Command::new(&shell[0]);
6563        cmd.quiet();
6564        cmd.args(&shell[1..])
6565            .arg(command)
6566            .current_dir(cwd)
6567            .stdin(std::process::Stdio::null())
6568            .stdout(std::process::Stdio::piped())
6569            .stderr(std::process::Stdio::piped())
6570            .kill_on_drop(true);
6571        let spawned = cmd.spawn();
6572        let (code, body) = match spawned {
6573            Ok(child) => {
6574                // Captured before the child is consumed below: `kill_on_drop`
6575                // only *asks* the process to die when the timeout branch
6576                // drops it, and the pid is the only way anyone downstream can
6577                // later check whether that request actually took.
6578                let pid = child.id();
6579                match tokio::time::timeout(timeout, child.wait_with_output()).await {
6580                    Ok(Ok(o)) => {
6581                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
6582                        body.push_str(&String::from_utf8_lossy(&o.stderr));
6583                        (o.status.code(), body)
6584                    }
6585                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
6586                    Err(_) => {
6587                        if let Some(pid) = pid {
6588                            timed_out_pids.push(pid);
6589                        }
6590                        (None, format!("timed out after {}s", timeout.as_secs()))
6591                    }
6592                }
6593            }
6594            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
6595        };
6596        out.push(CommandOutcome {
6597            command: command.clone(),
6598            code,
6599            output_tail: tail(&body, OUTPUT_TAIL),
6600            duration_ms: started.elapsed().as_millis() as u64,
6601            resource_blocked: false,
6602        });
6603    }
6604    state.task_finished(task);
6605    if let Err(e) = state.save() {
6606        tracing::warn!("could not persist the end of {task}: {e:#}");
6607    }
6608    (out, timed_out_pids)
6609}
6610
6611/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
6612/// section (`report::run`) and in the `merge` event this node records — for
6613/// the operator to run by hand.
6614///
6615/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
6616/// branch whose ruleset forbids merge commits (GitHub's "must not contain
6617/// merge commits", or "require linear history") rejects the push a `--no-ff`
6618/// merge would produce, which is exactly the guidance this function replaces.
6619/// `message`'s first line becomes the squash commit's subject, matching the
6620/// note `report::run` prints alongside this command — see that function for
6621/// why an explicit subject is not optional there.
6622fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
6623    let repo = repo.display();
6624    match style {
6625        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
6626        MergeStyle::Squash => {
6627            // The subject sits inside double quotes, and a title an agent
6628            // wrote may carry the characters that break out of them.
6629            let subject = message
6630                .lines()
6631                .next()
6632                .unwrap_or(branch)
6633                .replace(['\\', '"', '$', '`'], "");
6634            format!(
6635                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
6636            )
6637        }
6638        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
6639    }
6640}
6641
6642/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
6643/// under the hood, rejects a `title` over 256 characters and the whole
6644/// command fails — no PR at all, for a run whose body was otherwise fine
6645/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
6646/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
6647/// always how GitHub counts, plus one character for the trailing ellipsis
6648/// `title_from` may add. It is a margin, not a guarantee — a title packed
6649/// with multi-unit characters could still in principle land close to the
6650/// edge, but a real task title's occasional emoji or accented letter fits
6651/// comfortably inside it.
6652const PR_TITLE_MAX: usize = 240;
6653
6654/// What `merge = "pr"` (and the merge commit of the other modes) says about a
6655/// change: a title and a body describing what was *implemented*, not the task
6656/// that asked for it. A task reads as a request; a reader of the merged
6657/// history wants the change.
6658struct PrMessage {
6659    title: String,
6660    body: String,
6661}
6662
6663impl PrMessage {
6664    /// Title, blank line, body. The first line is the squash/merge commit
6665    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
6666    /// has to stay one sensible line.
6667    fn commit_message(&self) -> String {
6668        format!("{}\n\n{}", self.title, self.body)
6669    }
6670}
6671
6672/// The text after a leading `TITLE:` (any case) on `line`.
6673fn title_marker(line: &str) -> Option<&str> {
6674    let line = line.trim();
6675    let head = line.get(..6)?;
6676    head.eq_ignore_ascii_case("title:")
6677        .then(|| line[6..].trim())
6678}
6679
6680/// The implementer's own one-line title: the `TITLE:` line the implement
6681/// prompt asks for at the top of its SUMMARY. Candidate commits are all
6682/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
6683/// source, and a title that says as much is refused here too.
6684fn summary_title(summary: &str) -> Option<String> {
6685    let first = summary.lines().find(|l| !l.trim().is_empty())?;
6686    let raw = title_marker(first)?;
6687    if raw.is_empty() {
6688        return None;
6689    }
6690    let title = queue::title_from(raw, PR_TITLE_MAX);
6691    let lower = title.to_ascii_lowercase();
6692    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
6693        return None;
6694    }
6695    Some(title)
6696}
6697
6698/// `summary` without its `TITLE:` line, which the pull request title already
6699/// carries.
6700fn summary_without_title(summary: &str) -> String {
6701    let mut lines = summary.trim().lines().peekable();
6702    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
6703        lines.next();
6704    }
6705    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
6706}
6707
6708/// The pull request title and body for the winning candidate.
6709///
6710/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
6711/// the task's own opening line via [`queue::title_from`] when there is none.
6712/// `state.instruction` can open with blank lines (`task_text` only rejects a
6713/// body that is blank *entirely*), which `title_from` skips.
6714///
6715/// Body: the implementer's summary and the fixer's notes, then — when the
6716/// winning review round was not clean — the findings still open and whatever
6717/// the fixer declined, so `merge = "pr"` hands the reader the same material
6718/// `magi show` does. The task follows inside a collapsed block, and the
6719/// footer repeats the run and candidate as plain tags for a reader holding
6720/// only the merged commit or the PR body.
6721fn pr_message(state: &RunState, winner: char) -> PrMessage {
6722    let summary = state
6723        .candidates
6724        .iter()
6725        .find(|c| c.label == winner)
6726        .map(|c| c.summary.as_str())
6727        .unwrap_or_default();
6728    // The fallback is the operator's own words and may not be English; GitHub
6729    // text always is, so a non-English task gets a neutral title instead.
6730    let title = summary_title(summary).unwrap_or_else(|| {
6731        let t = queue::title_from(&state.instruction, PR_TITLE_MAX);
6732        if t.is_ascii() && t.chars().any(|c| c.is_ascii_alphabetic()) {
6733            t
6734        } else {
6735            format!(
6736                "chore: land candidate {} of run {}",
6737                winner.to_ascii_uppercase(),
6738                state.id
6739            )
6740        }
6741    });
6742
6743    let mut body = String::new();
6744    let what = summary_without_title(summary);
6745    if !what.is_empty() {
6746        body.push_str("## Summary\n\n");
6747        body.push_str(&what);
6748        body.push_str("\n\n");
6749    }
6750
6751    let fix = state.reviews.last().and_then(|r| r.fix.as_ref());
6752    if let Some(fix) = fix
6753        && !fix.notes.trim().is_empty()
6754    {
6755        body.push_str("## Review fixes\n\n");
6756        body.push_str(fix.notes.trim());
6757        body.push_str("\n\n");
6758    }
6759
6760    let open = state.open_findings();
6761    if !open.is_empty() {
6762        body.push_str("## Open review findings\n\n");
6763        for f in &open {
6764            body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
6765        }
6766        body.push('\n');
6767    }
6768
6769    if let Some(fix) = fix
6770        && !fix.rejected.is_empty()
6771    {
6772        body.push_str("## Declined by the fixer\n\n");
6773        for r in &fix.rejected {
6774            body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
6775        }
6776        body.push('\n');
6777    }
6778
6779    let task = state.instruction.trim();
6780    let task = if task.is_empty() {
6781        "(empty task)"
6782    } else {
6783        task
6784    };
6785    body.push_str(&format!(
6786        "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
6787        task.replace("</details>", "&lt;/details&gt;")
6788    ));
6789
6790    body.push_str(&format!(
6791        "\n---\nmagi:run/{} magi:candidate-{}\n",
6792        state.id,
6793        winner.to_ascii_lowercase()
6794    ));
6795
6796    // Prompts are advisory; this is the enforced half of the confidentiality
6797    // rule, and it covers the verbatim task in <details> too.
6798    let id = crate::scrub::Identity::current();
6799    PrMessage {
6800        title: crate::scrub::scrub(&title, &id),
6801        body: crate::scrub::scrub(&body, &id),
6802    }
6803}
6804
6805/// `gh pr create`, returning the PR url.
6806async fn gh_pr_create(
6807    cwd: &Path,
6808    base: &str,
6809    head: &str,
6810    title: &str,
6811    body: &str,
6812) -> Result<String> {
6813    let out = tokio::process::Command::new("gh")
6814        .args([
6815            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
6816        ])
6817        .current_dir(cwd)
6818        .quiet()
6819        .stdin(std::process::Stdio::null())
6820        .output()
6821        .await
6822        .context("spawn gh")?;
6823    if out.status.success() {
6824        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
6825    } else {
6826        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
6827    }
6828}
6829
6830/// Tear a run's worktrees and branches down.
6831///
6832/// `home` is where the updated `run.json` is saved (via
6833/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
6834/// a housekeeping pass already has its own honest `home` handed to it, and
6835/// falling through to the global here would write back through whichever
6836/// directory some other process or test pinned into that `OnceLock` first,
6837/// not the one the caller actually resolved its `runs` and `state` from.
6838pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
6839    let repo = state.repo.clone();
6840    let root = state.worktree_root();
6841    let winner = state.tally.as_ref().map(|t| t.winner);
6842    let mut removed = Vec::new();
6843
6844    for i in 0..state.candidates.len() {
6845        let c = state.candidates[i].clone();
6846        let is_winner = Some(c.label) == winner;
6847        if is_winner && !drop_winner {
6848            continue;
6849        }
6850        if c.worktree.exists() {
6851            git::worktree_remove(&repo, &c.worktree).await.ok();
6852            removed.push(c.worktree.to_string_lossy().into_owned());
6853        }
6854        // A branch handed to a later run (and its pull request) is not this
6855        // run's to delete.
6856        let handed_over = state.released_branches.contains(&c.branch);
6857        if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
6858            git::branch_delete(&repo, &c.branch).await.ok();
6859            removed.push(c.branch.clone());
6860        }
6861        state.candidates[i].folded = true;
6862    }
6863
6864    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
6865        let path = name.path();
6866        let keep = !drop_winner
6867            && winner.is_some_and(|w| {
6868                path.file_name()
6869                    .is_some_and(|n| n == format!("cand-{w}").as_str())
6870            });
6871        if keep {
6872            continue;
6873        }
6874        git::worktree_remove(&repo, &path).await.ok();
6875        removed.push(path.to_string_lossy().into_owned());
6876    }
6877
6878    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
6879    // judge worktrees, so once the loop above has cleared all of them out,
6880    // the parent is a bare directory nobody else was ever going to remove -
6881    // git only ever managed what was inside it. Left alone, one of these
6882    // accumulates per fully-folded run; the operator's own machine had 74.
6883    // `remove_if_empty` re-checks rather than assuming: a run whose winner
6884    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
6885    // so does anything a run never claimed that happens to share the bay.
6886    remove_if_empty(&root);
6887
6888    if state.enabled_worktree_config && drop_winner {
6889        // A release, not a raw disable: some sibling run in this repository
6890        // may still hold its own reference (see `git::acquire_worktree_config`),
6891        // and only the last release actually turns the setting back off.
6892        git::release_worktree_config(&repo).await.ok();
6893        state.enabled_worktree_config = false;
6894    }
6895    state.save_under(home)?;
6896    Ok(removed)
6897}
6898
6899/// Remove `dir` if it exists and has nothing in it.
6900///
6901/// Best-effort and silent by design: a directory that is not empty (a run
6902/// whose winner is still parked there, a stray file some other process left)
6903/// is exactly the case this must refuse, and a directory that is already gone
6904/// is not a failure worth reporting either. `std::fs::remove_dir` itself
6905/// already refuses a non-empty directory, so the emptiness check below is
6906/// belt, not suspenders - it is what keeps this from ever attempting the
6907/// removal in the case that matters, rather than trusting `remove_dir`'s
6908/// error path to have no side effects if it ever changed.
6909fn remove_if_empty(dir: &Path) {
6910    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
6911        std::fs::remove_dir(dir).ok();
6912    }
6913}
6914
6915/// Severity of the worst open finding in the last review round, for reporting.
6916pub fn worst_open(state: &RunState) -> Option<Severity> {
6917    state
6918        .reviews
6919        .last()?
6920        .reviews
6921        .iter()
6922        .flat_map(|r| r.findings.iter())
6923        .map(|f| f.severity)
6924        .max()
6925}
6926
6927#[cfg(test)]
6928mod tests {
6929    use super::*;
6930    use crate::run::GateStatus;
6931    use std::collections::BTreeMap;
6932    use std::time::Duration;
6933
6934    fn conductor() -> AgentSpec {
6935        AgentSpec {
6936            id: "conductor".to_owned(),
6937            kind: crate::config::AgentKind::Command,
6938            model: None,
6939            command: vec!["true".to_owned()],
6940            extra_args: Vec::new(),
6941            env: BTreeMap::new(),
6942            prompt_delivery: None,
6943        }
6944    }
6945
6946    fn spec(id: &str) -> AgentSpec {
6947        AgentSpec {
6948            id: id.to_owned(),
6949            kind: crate::config::AgentKind::Command,
6950            model: None,
6951            command: vec!["true".to_owned()],
6952            extra_args: Vec::new(),
6953            env: BTreeMap::new(),
6954            prompt_delivery: None,
6955        }
6956    }
6957
6958    // `next_untried_implementer` is the property `resume_quota_losses`'s own
6959    // fallback loop depends on to terminate: it must walk forward from the
6960    // seat's own position, never restart at the front of the roster, and it
6961    // must never hand back an id already tried, however many times that id
6962    // happens to appear.
6963
6964    #[test]
6965    fn next_untried_implementer_walks_forward_from_the_seats_own_position() {
6966        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6967        let tried = BTreeSet::from(["beta".to_owned()]);
6968        // beta sits at index 1; the next candidate is gamma, never alpha —
6969        // which is very likely a different candidate slot's own agent.
6970        let next = next_untried_implementer(&roster, 1, &tried);
6971        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
6972    }
6973
6974    #[test]
6975    fn next_untried_implementer_does_not_wrap_back_past_its_own_start() {
6976        let roster = vec![spec("alpha"), spec("beta")];
6977        let tried = BTreeSet::from(["beta".to_owned()]);
6978        // beta is the roster's last entry: nothing follows it, and alpha —
6979        // earlier in the roster, almost certainly a different candidate
6980        // slot's own agent — must not be reached by wrapping back to it.
6981        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6982    }
6983
6984    #[test]
6985    fn next_untried_implementer_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
6986        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6987        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
6988        // beta (index 1) and gamma (index 2, the only entry after it) have
6989        // both been tried; alpha (index 0) never has, but it comes before
6990        // beta's own position, so there is nothing further for this seat.
6991        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6992    }
6993
6994    #[test]
6995    fn next_untried_implementer_skips_ids_already_tried_even_when_duplicated() {
6996        let roster = vec![spec("a"), spec("a"), spec("b")];
6997        let tried = BTreeSet::from(["a".to_owned()]);
6998        let next = next_untried_implementer(&roster, 0, &tried);
6999        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
7000    }
7001
7002    #[test]
7003    fn next_untried_implementer_returns_none_once_every_id_is_tried() {
7004        let roster = vec![spec("a"), spec("b")];
7005        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
7006        assert!(next_untried_implementer(&roster, 0, &tried).is_none());
7007    }
7008
7009    #[test]
7010    fn remove_if_empty_only_ever_takes_a_bare_directory() {
7011        let dir = tempfile::tempdir().unwrap();
7012        let bay = dir.path().join("ffff");
7013
7014        // Not there yet: nothing to do, nothing to panic on.
7015        remove_if_empty(&bay);
7016        assert!(!bay.exists());
7017
7018        // Something still inside - the winner's worktree, or a stray file -
7019        // keeps the directory standing.
7020        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
7021        remove_if_empty(&bay);
7022        assert!(bay.exists(), "non-empty directory must survive");
7023
7024        // Once the last entry is gone, so is the directory itself.
7025        std::fs::remove_dir(bay.join("cand-A")).unwrap();
7026        remove_if_empty(&bay);
7027        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
7028    }
7029
7030    // `round_is_clean` is the exact decision this task fixed: a round with a
7031    // seat that never answered must not read the same as a round every seat
7032    // actually reviewed. These are deterministic and process-free by design —
7033    // the equivalent end-to-end check (a real reviewer timing out under a
7034    // live graph run) is a genuine race against wall-clock contention, and a
7035    // spawn slow enough to blow even a generous budget under a loaded test
7036    // run must not turn this specific regression check flaky.
7037
7038    #[test]
7039    fn a_full_panel_that_found_nothing_is_clean() {
7040        assert!(round_is_clean(
7041            0,
7042            true,
7043            2,
7044            2,
7045            0,
7046            IncompleteReviewPolicy::Block
7047        ));
7048    }
7049
7050    #[test]
7051    fn a_missing_seat_is_never_clean_under_the_default_policy() {
7052        assert!(!round_is_clean(
7053            0,
7054            true,
7055            1,
7056            2,
7057            0,
7058            IncompleteReviewPolicy::Block
7059        ));
7060    }
7061
7062    #[test]
7063    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
7064        assert!(!round_is_clean(
7065            1,
7066            true,
7067            1,
7068            2,
7069            0,
7070            IncompleteReviewPolicy::Warn
7071        ));
7072    }
7073
7074    #[test]
7075    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
7076        assert!(round_is_clean(
7077            0,
7078            true,
7079            1,
7080            2,
7081            0,
7082            IncompleteReviewPolicy::Warn
7083        ));
7084    }
7085
7086    #[test]
7087    fn a_full_panel_with_an_open_finding_is_not_clean() {
7088        assert!(!round_is_clean(
7089            1,
7090            true,
7091            2,
7092            2,
7093            0,
7094            IncompleteReviewPolicy::Block
7095        ));
7096    }
7097
7098    #[test]
7099    fn a_full_panel_with_a_red_e2e_is_not_clean() {
7100        assert!(!round_is_clean(
7101            0,
7102            false,
7103            2,
7104            2,
7105            0,
7106            IncompleteReviewPolicy::Block
7107        ));
7108    }
7109
7110    // The stall this task closes: under the default `block` policy, a seat
7111    // missing only because it was rate limited must not force a wait for a
7112    // session limit that will not lift by the next round. `round_is_clean`
7113    // is where that quorum carve-out lives; the review loop around it never
7114    // changes what a reviewer's vote or a finding's severity means.
7115
7116    #[test]
7117    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
7118        // 1 of 2 answered, and the one missing was quota'd — the exact
7119        // "review-2 rate limited (quota)" shape from the field report.
7120        assert!(round_is_clean(
7121            0,
7122            true,
7123            1,
7124            2,
7125            1,
7126            IncompleteReviewPolicy::Block
7127        ));
7128    }
7129
7130    #[test]
7131    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
7132        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
7133        // not a quota loss (`quota_missing` stays 0) — worth another try.
7134        assert!(!round_is_clean(
7135            0,
7136            true,
7137            1,
7138            2,
7139            0,
7140            IncompleteReviewPolicy::Block
7141        ));
7142    }
7143
7144    #[test]
7145    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
7146        assert!(!round_is_clean(
7147            1,
7148            true,
7149            1,
7150            2,
7151            1,
7152            IncompleteReviewPolicy::Block
7153        ));
7154        assert!(!round_is_clean(
7155            0,
7156            false,
7157            1,
7158            2,
7159            1,
7160            IncompleteReviewPolicy::Block
7161        ));
7162    }
7163
7164    #[test]
7165    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
7166        // Every seat quota'd, nobody answered: there is no panel to decide
7167        // on, so this must fall through to the existing block-and-retry
7168        // fallback rather than call an unreviewed patch clean.
7169        assert!(!round_is_clean(
7170            0,
7171            true,
7172            0,
7173            2,
7174            2,
7175            IncompleteReviewPolicy::Block
7176        ));
7177    }
7178
7179    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
7180        CommandOutcome {
7181            command: "test".to_owned(),
7182            code,
7183            output_tail: String::new(),
7184            duration_ms: 0,
7185            resource_blocked,
7186        }
7187    }
7188
7189    #[test]
7190    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
7191        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
7192        assert!(
7193            !verify_inconclusive(&[outcome(Some(1), false)]),
7194            "an ordinary failure is still evidence about the patch"
7195        );
7196        assert!(verify_inconclusive(&[outcome(None, true)]));
7197        assert!(
7198            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
7199            "one inconclusive outcome taints the whole batch"
7200        );
7201        assert!(!verify_inconclusive(&[]));
7202    }
7203
7204    #[tokio::test]
7205    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
7206        // Alive for the first two checks, then dead - confirms the loop
7207        // actually re-polls rather than deciding once and sleeping out the
7208        // ceiling regardless.
7209        let calls = std::sync::atomic::AtomicUsize::new(0);
7210        let started = Instant::now();
7211        wait_for_pids_with(
7212            &[123],
7213            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
7214            Duration::from_millis(5),
7215            Duration::from_secs(5),
7216        )
7217        .await;
7218        assert!(
7219            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
7220            "must keep checking rather than deciding on the first answer"
7221        );
7222        assert!(
7223            started.elapsed() < Duration::from_secs(1),
7224            "must return the moment it is confirmed dead, not wait out the ceiling"
7225        );
7226    }
7227
7228    #[tokio::test]
7229    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
7230        let started = Instant::now();
7231        wait_for_pids_with(
7232            &[123],
7233            |_| true, // never reports dead
7234            Duration::from_millis(5),
7235            Duration::from_millis(30),
7236        )
7237        .await;
7238        let elapsed = started.elapsed();
7239        assert!(
7240            elapsed >= Duration::from_millis(30),
7241            "must not give up before its own ceiling: {elapsed:?}"
7242        );
7243        assert!(
7244            elapsed < Duration::from_secs(1),
7245            "must not wait past its own ceiling either: {elapsed:?}"
7246        );
7247    }
7248
7249    #[tokio::test]
7250    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
7251        let started = Instant::now();
7252        wait_for_pids_with(
7253            &[],
7254            |_| true,
7255            Duration::from_secs(5),
7256            Duration::from_secs(5),
7257        )
7258        .await;
7259        assert!(
7260            started.elapsed() < Duration::from_millis(200),
7261            "an empty pid list has nothing to confirm"
7262        );
7263    }
7264
7265    // `review_conclusion` is the exact decision the review hand-off task
7266    // fixed: a round budget spent (or a tree that stopped moving) must not
7267    // collapse into `Blocked` regardless of what verification actually
7268    // said. Deterministic and process-free for the same reason the
7269    // `round_is_clean` family above is.
7270    fn review_round(
7271        clean: bool,
7272        blocking: usize,
7273        answered: usize,
7274        expected: usize,
7275        progressed: bool,
7276        e2e_ok: bool,
7277    ) -> ReviewRound {
7278        ReviewRound {
7279            round: 1,
7280            head: "h".to_owned(),
7281            verified_head: None,
7282            verified_at: None,
7283            reviews: Vec::new(),
7284            e2e: vec![CommandOutcome {
7285                command: "test".to_owned(),
7286                code: Some(if e2e_ok { 0 } else { 1 }),
7287                output_tail: String::new(),
7288                duration_ms: 0,
7289                resource_blocked: false,
7290            }],
7291            verify_retried: false,
7292            e2e_deferred: false,
7293            e2e_defer_reason: None,
7294            fix: None,
7295            blocking,
7296            answered,
7297            expected,
7298            clean,
7299            progressed,
7300            vote_split: false,
7301            reconsideration: Vec::new(),
7302            verdict: None,
7303        }
7304    }
7305
7306    #[test]
7307    fn review_conclusion_is_none_when_nothing_has_run() {
7308        assert_eq!(review_conclusion(&[], 3), None);
7309    }
7310
7311    #[test]
7312    fn review_conclusion_is_none_while_rounds_remain() {
7313        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
7314        assert_eq!(review_conclusion(&rounds, 3), None);
7315    }
7316
7317    #[test]
7318    fn review_conclusion_is_gating_once_a_round_is_clean() {
7319        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
7320        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
7321    }
7322
7323    #[test]
7324    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
7325        let rounds = vec![
7326            review_round(false, 1, 2, 2, true, true),
7327            review_round(false, 1, 2, 2, true, true),
7328        ];
7329        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
7330    }
7331
7332    #[test]
7333    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
7334        let rounds = vec![
7335            review_round(false, 1, 2, 2, true, true),
7336            review_round(false, 1, 2, 2, true, false),
7337        ];
7338        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
7339    }
7340
7341    #[test]
7342    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
7343        // Magi never got a command to run against this round's own head — a
7344        // resource-blocked attempt, not a red one — so this must never
7345        // settle on `Blocked` the way a genuine e2e failure would. `None`
7346        // here is what tells `Runner::review_loop` to retry the check
7347        // itself rather than trust this cheap recomputation with a verdict
7348        // it cannot actually produce.
7349        let mut blocked = review_round(false, 1, 2, 2, true, false);
7350        blocked.e2e[0].resource_blocked = true;
7351        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
7352        assert_eq!(review_conclusion(&rounds, 2), None);
7353    }
7354
7355    #[test]
7356    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
7357        // Missing input, not a verified tree — never a hand-off candidate.
7358        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
7359        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
7360    }
7361
7362    #[test]
7363    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
7364        let rounds = vec![
7365            review_round(false, 1, 2, 2, false, true),
7366            review_round(false, 1, 2, 2, false, true),
7367        ];
7368        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
7369    }
7370
7371    fn secs(n: u64) -> Duration {
7372        Duration::from_secs(n)
7373    }
7374
7375    /// A throwaway repo with one commit on `main`, for tests that need `merge`
7376    /// to make real (and, if it runs at all, real*ly fail*) git calls.
7377    fn init_repo(dir: &Path) {
7378        let run = |args: &[&str]| {
7379            let out = std::process::Command::new("git")
7380                .args(args)
7381                .current_dir(dir)
7382                .quiet()
7383                .output()
7384                .expect("spawn git");
7385            assert!(
7386                out.status.success(),
7387                "git {args:?} failed: {}",
7388                String::from_utf8_lossy(&out.stderr)
7389            );
7390        };
7391        run(&["init", "-b", "main"]);
7392        run(&["config", "user.name", "magi test"]);
7393        run(&["config", "user.email", "magi@example.com"]);
7394        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
7395        run(&["add", "-A"]);
7396        run(&["commit", "-m", "init"]);
7397    }
7398
7399    // `settle_questions` is what closes the ghost the phone showed: a run's
7400    // seat asked something, the run then ended, and nothing was left to
7401    // abandon the question it left `open`. `HOME` is a process-wide
7402    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
7403    // first time it runs in the binary — every test below still reaches the
7404    // same directory whichever call won, and each gets its own run id from
7405    // `RunState::new`, so they never collide there.
7406    fn ask_test_home() {
7407        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
7408    }
7409
7410    /// A minimal, git-free `Runner` at a given status — `settle_questions`
7411    /// reads nothing else off it.
7412    fn runner_at(status: RunStatus) -> Runner {
7413        let mut state = RunState::new(
7414            PathBuf::from("/nonexistent/repo"),
7415            "main".to_owned(),
7416            "deadbeef".to_owned(),
7417            "task".to_owned(),
7418            Config::default(),
7419        );
7420        state.status = status;
7421        Runner {
7422            state,
7423            roles: ResolvedRoles {
7424                implementers: Vec::new(),
7425                judges: Vec::new(),
7426                reviewers: Vec::new(),
7427                fixer: None,
7428                conductor: conductor(),
7429                implementer_roster: Vec::new(),
7430            },
7431            sem: Arc::new(Semaphore::new(1)),
7432            pause: Pause::new(),
7433            interrupt: Pause::new(),
7434        }
7435    }
7436
7437    /// `park_here` folding in the reason `Pause::park_because` recorded -
7438    /// this is what lets an operator reading a run's events tell an
7439    /// interrupt-driven park from an ordinary shutdown park.
7440    #[test]
7441    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
7442        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7443        let mut runner = runner_at(RunStatus::Implementing);
7444        let interrupt = Pause::new();
7445        runner.watch_interrupt(interrupt.clone());
7446
7447        interrupt.park_because("task a1b2 asked to run first");
7448
7449        assert!(runner.park_here().expect("park_here"));
7450        assert!(runner.state.parked);
7451        let last = runner.state.events.last().expect("a park event");
7452        assert_eq!(last.node, "park");
7453        assert!(
7454            last.message.contains("task a1b2 asked to run first"),
7455            "expected the interrupt reason in {:?}",
7456            last.message
7457        );
7458    }
7459
7460    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
7461    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
7462    /// cleared) must not make a *different* run - one only watching its own,
7463    /// unshared interrupt `Pause` - see itself as parked. If a future change
7464    /// ever collapsed these back into one handle, the interrupt scheduler
7465    /// would park every run for the rest of the daemon's life, not just the
7466    /// one it meant to interrupt.
7467    #[test]
7468    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
7469        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7470        let mut runner = runner_at(RunStatus::Implementing);
7471        let shutdown = Pause::new();
7472        runner.on_pause(shutdown.clone());
7473        let interrupt = Pause::new();
7474        runner.watch_interrupt(interrupt.clone());
7475
7476        // Nobody has asked for anything yet.
7477        assert!(!runner.park_here().expect("park_here"));
7478        assert!(!runner.state.parked);
7479
7480        // Only the interrupt handle fires; the shutdown handle stays clear.
7481        interrupt.park_because("test");
7482        assert!(!shutdown.parked());
7483        assert!(runner.park_here().expect("park_here"));
7484    }
7485
7486    /// The property every prior attempt at this feature failed to pin down:
7487    /// asking a run to park while one of its nodes has a real, in-flight
7488    /// async operation running (an agent call, in production) must not cut
7489    /// that operation short. `park_here` is only ever consulted *between*
7490    /// `execute`'s node calls - see its own doc - so nothing inside a node
7491    /// can observe a park request until the node itself returns. This proves
7492    /// that structurally, with real `tokio` concurrency and a channel
7493    /// handshake (never a sleep, which would only prove "usually", not
7494    /// "cannot"): the "node" below reports that it has genuinely started,
7495    /// and only then is the park requested; the node still has to be told to
7496    /// finish before `park_here` is ever called, exactly mirroring every
7497    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
7498    /// in `execute`.
7499    #[tokio::test]
7500    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
7501        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7502        let mut runner = runner_at(RunStatus::Implementing);
7503        let interrupt = Pause::new();
7504        runner.watch_interrupt(interrupt.clone());
7505
7506        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
7507        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
7508
7509        // Stands in for one node's in-flight agent call: it proves it has
7510        // genuinely started, then blocks - exactly as a spawned CLI process
7511        // does - until told to finish.
7512        let node = async move {
7513            started_tx.send(()).expect("send started");
7514            finish_rx.await.expect("recv finish");
7515            "node finished"
7516        };
7517
7518        let interrupter = async move {
7519            started_rx.await.expect("recv started");
7520            // The call is now genuinely in flight. Ask it to park.
7521            interrupt.park_because("higher-priority task waiting");
7522            // Nothing the node does can observe this yet - there is no
7523            // check inside it, by construction - so let the executor run
7524            // anything pending and then let the node finish on its own.
7525            tokio::task::yield_now().await;
7526            finish_tx.send(()).expect("send finish");
7527        };
7528
7529        let (node_result, ()) = tokio::join!(node, interrupter);
7530        assert_eq!(
7531            node_result, "node finished",
7532            "the in-flight call ran to completion"
7533        );
7534
7535        // Only now, at the boundary the real `execute` would check right
7536        // after this node, does the park take effect.
7537        assert!(runner.park_here().expect("park_here"));
7538        assert!(runner.state.parked);
7539    }
7540
7541    /// A run parked mid-competition carries every field it had accumulated
7542    /// through the exact same disk round-trip an ordinary resume uses -
7543    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
7544    /// Nothing about parking for an interrupt is a special case of that path;
7545    /// this is what proves it rather than assuming it.
7546    #[test]
7547    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
7548        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7549        let mut runner = runner_at(RunStatus::Judging);
7550        // `Runner::resume` re-resolves roles from the saved config, which
7551        // refuses an empty roster - give it the same minimal one `conductor`
7552        // itself uses.
7553        runner.state.config.agents = vec![conductor()];
7554        runner.state.candidates = vec![Candidate {
7555            index: 0,
7556            label: 'A',
7557            agent: "alpha".to_owned(),
7558            branch: "magi/x/A".to_owned(),
7559            worktree: PathBuf::from("/nonexistent/worktree"),
7560            summary: "did the thing".to_owned(),
7561            stat: "1 file changed".to_owned(),
7562            files: 1,
7563            commits: 1,
7564            empty: false,
7565            failed: None,
7566            verified_noop: None,
7567            duration_ms: 1234,
7568            folded: false,
7569        }];
7570        let run_id = runner.state.id.clone();
7571
7572        let interrupt = Pause::new();
7573        runner.watch_interrupt(interrupt.clone());
7574        interrupt.park_because("task c3d4 asked to run first");
7575        assert!(runner.park_here().expect("park_here"));
7576
7577        let resumed = Runner::resume(&run_id).expect("resume");
7578        assert_eq!(resumed.state.candidates.len(), 1);
7579        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
7580        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
7581        assert_eq!(resumed.state.status, runner.state.status);
7582        assert!(
7583            resumed.state.parked,
7584            "still parked until `execute` actually walks the graph again"
7585        );
7586        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
7587    }
7588
7589    /// A fresh open question on `run`, stored and handed back for assertions.
7590    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
7591        let mut q = ask::Question::new(
7592            run.to_owned(),
7593            "implement".to_owned(),
7594            "impl-A".to_owned(),
7595            "Which storage backend should the cache use?".to_owned(),
7596            String::new(),
7597            vec!["SQLite".to_owned(), "Redis".to_owned()],
7598        );
7599        store.put(&mut q).unwrap();
7600        q
7601    }
7602
7603    #[test]
7604    fn a_failed_runs_open_question_is_abandoned() {
7605        ask_test_home();
7606        let store = ask::Questions::open();
7607        let mut runner = runner_at(RunStatus::Failed);
7608        let run = runner.state.id.clone();
7609        let q = ask_open_question(&store, &run);
7610
7611        runner.settle_questions();
7612
7613        let back = store.get(&q.id).unwrap();
7614        assert!(
7615            !back.status.open(),
7616            "the seat that asked died with the run; nobody is left to read an answer"
7617        );
7618        assert!(
7619            back.detail.contains(&run) && back.detail.contains("failed"),
7620            "the reason names what the run became, not just that it is gone: {}",
7621            back.detail
7622        );
7623    }
7624
7625    #[test]
7626    fn a_merged_runs_open_question_is_abandoned_too() {
7627        ask_test_home();
7628        let store = ask::Questions::open();
7629        // A run that finishes cleanly still leaves nobody to read an answer -
7630        // this is not only a failure-path cleanup.
7631        for status in [RunStatus::Merged, RunStatus::Ready] {
7632            let mut runner = runner_at(status);
7633            let run = runner.state.id.clone();
7634            let q = ask_open_question(&store, &run);
7635
7636            runner.settle_questions();
7637
7638            let back = store.get(&q.id).unwrap();
7639            assert!(
7640                !back.status.open(),
7641                "{status:?} run's question must not outlive the run"
7642            );
7643        }
7644    }
7645
7646    #[test]
7647    fn a_still_resumable_runs_open_question_is_left_alone() {
7648        ask_test_home();
7649        let store = ask::Questions::open();
7650        // `Blocked` and `Stalled` can still be resumed — the candidates, the
7651        // review round and the seat sessions are all still on disk — so a
7652        // question asked mid-round may yet get a real answer from a real
7653        // resume. Sweeping it here would be exactly the failure mode this
7654        // whole feature exists to avoid on the other side.
7655        for status in [RunStatus::Blocked, RunStatus::Stalled] {
7656            let mut runner = runner_at(status);
7657            let run = runner.state.id.clone();
7658            let q = ask_open_question(&store, &run);
7659
7660            runner.settle_questions();
7661
7662            let back = store.get(&q.id).unwrap();
7663            assert!(
7664                back.status.open(),
7665                "{status:?} is still alive; the question must still be waiting"
7666            );
7667        }
7668    }
7669
7670    #[test]
7671    fn settle_questions_never_touches_an_already_answered_question() {
7672        ask_test_home();
7673        let store = ask::Questions::open();
7674        let mut runner = runner_at(RunStatus::Failed);
7675        let run = runner.state.id.clone();
7676        let mut q = ask_open_question(&store, &run);
7677        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
7678            .unwrap();
7679        store.put(&mut q).unwrap();
7680
7681        // Called twice, the way a crash-recovered daemon reclaim and the
7682        // graph's own cleanup both can for the same run — `abandon_for_run`
7683        // only ever touches what is still open, so this must be inert both
7684        // times, not merely the second.
7685        runner.settle_questions();
7686        runner.settle_questions();
7687
7688        let back = store.get(&q.id).unwrap();
7689        assert_eq!(
7690            back.status,
7691            ask::QuestionStatus::Answered,
7692            "a real answer is a decision on record, never overwritten by a sweep"
7693        );
7694    }
7695
7696    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
7697    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
7698    /// without merging, whose winner is still the operator's answer to read.
7699    /// Nothing previously called `fold_run` itself with a real `tally`, so
7700    /// this is the first test to pin down the one distinction the whole
7701    /// automatic-fold feature depends on: the winner's worktree and branch
7702    /// must survive, everything else sharing the run's worktree bay - a
7703    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
7704    /// second sweep treats every non-winner directory under the bay alike -
7705    /// must not.
7706    #[tokio::test]
7707    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
7708        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
7709        let tmp = tempfile::tempdir().expect("tempdir");
7710        let repo = tmp.path().join("repo");
7711        std::fs::create_dir_all(&repo).unwrap();
7712        init_repo(&repo);
7713
7714        let mut config = Config::default();
7715        config.graph.worktree_root = Some(tmp.path().join("wt"));
7716
7717        let mut state = RunState::new(
7718            repo.clone(),
7719            "main".to_owned(),
7720            "deadbeef".to_owned(),
7721            "task".to_owned(),
7722            config,
7723        );
7724        let root = state.worktree_root();
7725        let wt_a = root.join("cand-A");
7726        let wt_b = root.join("cand-B");
7727        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
7728            .await
7729            .expect("worktree A");
7730        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
7731            .await
7732            .expect("worktree B");
7733
7734        state.candidates = vec![
7735            Candidate {
7736                index: 0,
7737                label: 'A',
7738                agent: "alpha".to_owned(),
7739                branch: "magi/x/A".to_owned(),
7740                worktree: wt_a.clone(),
7741                summary: String::new(),
7742                stat: String::new(),
7743                files: 0,
7744                commits: 0,
7745                empty: false,
7746                failed: None,
7747                verified_noop: None,
7748                duration_ms: 0,
7749                folded: false,
7750            },
7751            Candidate {
7752                index: 1,
7753                label: 'B',
7754                agent: "beta".to_owned(),
7755                branch: "magi/x/B".to_owned(),
7756                worktree: wt_b.clone(),
7757                summary: String::new(),
7758                stat: String::new(),
7759                files: 0,
7760                commits: 0,
7761                empty: false,
7762                failed: None,
7763                verified_noop: None,
7764                duration_ms: 0,
7765                folded: false,
7766            },
7767        ];
7768        state.tally = Some(Tally {
7769            first_choice: BTreeMap::from([('A', 1)]),
7770            borda: BTreeMap::new(),
7771            winner: 'A',
7772            rankings: 1,
7773            unanimous_initial: true,
7774            deliberated: false,
7775            changed_votes: 0,
7776            unanimous_final: true,
7777            tie_break: None,
7778            judges: 1,
7779            present: 1,
7780            quorum: 1,
7781            met_quorum: true,
7782            uncontested: None,
7783        });
7784        state.status = RunStatus::Ready;
7785
7786        fold_run(&mut state, false, &crate::run::home())
7787            .await
7788            .expect("fold_run");
7789
7790        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
7791        assert!(
7792            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7793            "the unmerged winner's branch survives"
7794        );
7795        assert!(
7796            !state.candidates[0].folded,
7797            "the winner is not marked folded"
7798        );
7799
7800        assert!(!wt_b.exists(), "the loser's worktree is removed");
7801        assert!(
7802            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
7803            "the loser's branch is removed"
7804        );
7805        assert!(state.candidates[1].folded, "the loser is marked folded");
7806    }
7807
7808    /// A branch handed to a later run is that run's (and its pull request's):
7809    /// folding the run that released it must not delete it.
7810    #[tokio::test]
7811    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
7812        let tmp = tempfile::tempdir().expect("tempdir");
7813        let repo = tmp.path().join("repo");
7814        std::fs::create_dir_all(&repo).unwrap();
7815        init_repo(&repo);
7816        let home = tmp.path().join("home");
7817
7818        let mut config = Config::default();
7819        config.graph.worktree_root = Some(tmp.path().join("wt"));
7820        let mut state = RunState::new(
7821            repo.clone(),
7822            "main".to_owned(),
7823            "deadbeef".to_owned(),
7824            "task".to_owned(),
7825            config,
7826        );
7827        // The worktree is already gone (released); the branch survives.
7828        git::git(&repo, &["branch", "magi/x/A", "main"])
7829            .await
7830            .expect("branch");
7831        state.candidates = vec![Candidate {
7832            index: 0,
7833            label: 'A',
7834            agent: "alpha".to_owned(),
7835            branch: "magi/x/A".to_owned(),
7836            worktree: state.worktree_root().join("cand-A"),
7837            summary: String::new(),
7838            stat: String::new(),
7839            files: 0,
7840            commits: 0,
7841            empty: false,
7842            failed: None,
7843            verified_noop: None,
7844            duration_ms: 0,
7845            folded: true,
7846        }];
7847        state.released_to = Some("20260901-000000-new1".to_owned());
7848        state.released_branches = vec!["magi/x/A".to_owned()];
7849
7850        fold_run(&mut state, true, &home).await.expect("fold_run");
7851
7852        assert!(
7853            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7854            "the handed-over branch survives a fold"
7855        );
7856    }
7857
7858    /// `status == Ready` used to be read as "this is the harmless
7859    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
7860    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
7861    /// run's PR was closed without merging — and reentering `merge` with
7862    /// `mode` still `Pr` does not know the difference, so it pushed and
7863    /// opened a second pull request. `mode == Local` reproduces the same
7864    /// blind spot without a network call: reentry must not attempt another
7865    /// git merge once this node has already recorded an outcome.
7866    #[tokio::test]
7867    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
7868        let tmp = tempfile::tempdir().expect("tempdir");
7869        let repo = tmp.path().join("repo");
7870        std::fs::create_dir_all(&repo).unwrap();
7871        init_repo(&repo);
7872
7873        let mut config = Config::default();
7874        config.merge.mode = MergeMode::Local;
7875
7876        let mut state = RunState::new(
7877            repo.clone(),
7878            "main".to_owned(),
7879            "deadbeef".to_owned(),
7880            "task".to_owned(),
7881            config,
7882        );
7883        state.candidates = vec![Candidate {
7884            index: 0,
7885            label: 'A',
7886            agent: "alpha".to_owned(),
7887            branch: "does-not-exist".to_owned(),
7888            worktree: repo.clone(),
7889            summary: String::new(),
7890            stat: String::new(),
7891            files: 0,
7892            commits: 0,
7893            empty: false,
7894            failed: None,
7895            verified_noop: None,
7896            duration_ms: 0,
7897            folded: false,
7898        }];
7899        state.tally = Some(Tally {
7900            first_choice: BTreeMap::from([('A', 1)]),
7901            borda: BTreeMap::new(),
7902            winner: 'A',
7903            rankings: 1,
7904            unanimous_initial: true,
7905            deliberated: false,
7906            changed_votes: 0,
7907            unanimous_final: true,
7908            tie_break: None,
7909            judges: 0,
7910            present: 0,
7911            quorum: 0,
7912            met_quorum: true,
7913            uncontested: Some("only candidate A produced a change".to_owned()),
7914        });
7915        state.reviews = vec![ReviewRound {
7916            round: 1,
7917            head: "deadbeef".to_owned(),
7918            verified_head: None,
7919            verified_at: None,
7920            reviews: Vec::new(),
7921            e2e: Vec::new(),
7922            fix: None,
7923            blocking: 0,
7924            answered: 0,
7925            expected: 0,
7926            clean: true,
7927            verify_retried: false,
7928            e2e_deferred: false,
7929            e2e_defer_reason: None,
7930            progressed: false,
7931            vote_split: false,
7932            reconsideration: Vec::new(),
7933            verdict: None,
7934        }];
7935        state.gate = vec![CommandOutcome {
7936            command: "test".to_owned(),
7937            code: Some(0),
7938            output_tail: String::new(),
7939            duration_ms: 0,
7940            resource_blocked: false,
7941        }];
7942        state.gate_ran = true;
7943        // Reached its conclusion already — e.g. `land` closing the PR without
7944        // merging it, which (like the honest `MergeMode::None` path) leaves
7945        // `status` at `Ready`. The recorded outcome is what actually marks
7946        // this node done.
7947        state.status = RunStatus::Ready;
7948        state.merge = Some(MergeOutcome {
7949            mode: MergeMode::Local,
7950            ok: false,
7951            detail: "already concluded".to_owned(),
7952        });
7953
7954        let mut runner = Runner {
7955            state,
7956            roles: ResolvedRoles {
7957                implementers: Vec::new(),
7958                judges: Vec::new(),
7959                reviewers: Vec::new(),
7960                fixer: None,
7961                conductor: conductor(),
7962                implementer_roster: Vec::new(),
7963            },
7964            sem: Arc::new(Semaphore::new(1)),
7965            pause: Pause::new(),
7966            interrupt: Pause::new(),
7967        };
7968
7969        runner.merge().await.expect("merge");
7970
7971        assert_eq!(
7972            runner.state.status,
7973            RunStatus::Ready,
7974            "a concluded run's status must not change on reentry"
7975        );
7976        assert_eq!(
7977            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
7978            Some("already concluded"),
7979            "merge must not run again once the node already recorded an outcome"
7980        );
7981    }
7982
7983    /// `gate` leaves `state.gate_ran` false both before it has ever run and
7984    /// when its last attempt was resource-blocked (the shared build cache
7985    /// could not be acquired or confirmed fresh in time - see
7986    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
7987    /// `Vec` this also leaves behind used to read as "nothing failed" and let
7988    /// a run merge a tree the gate never actually checked - exactly the case
7989    /// a contended cache produces on every retry until it clears. `merge`
7990    /// must refuse until `gate` has actually recorded an attempt.
7991    #[tokio::test]
7992    async fn merge_refuses_a_gate_that_has_not_actually_run() {
7993        let tmp = tempfile::tempdir().expect("tempdir");
7994        let repo = tmp.path().join("repo");
7995        std::fs::create_dir_all(&repo).unwrap();
7996        init_repo(&repo);
7997
7998        let mut config = Config::default();
7999        config.merge.mode = MergeMode::Local;
8000
8001        let mut state = RunState::new(
8002            repo.clone(),
8003            "main".to_owned(),
8004            "deadbeef".to_owned(),
8005            "task".to_owned(),
8006            config,
8007        );
8008        state.candidates = vec![Candidate {
8009            index: 0,
8010            label: 'A',
8011            agent: "alpha".to_owned(),
8012            branch: "does-not-exist".to_owned(),
8013            worktree: repo.clone(),
8014            summary: String::new(),
8015            stat: String::new(),
8016            files: 0,
8017            commits: 0,
8018            empty: false,
8019            failed: None,
8020            verified_noop: None,
8021            duration_ms: 0,
8022            folded: false,
8023        }];
8024        state.tally = Some(Tally {
8025            first_choice: BTreeMap::from([('A', 1)]),
8026            borda: BTreeMap::new(),
8027            winner: 'A',
8028            rankings: 1,
8029            unanimous_initial: true,
8030            deliberated: false,
8031            changed_votes: 0,
8032            unanimous_final: true,
8033            tie_break: None,
8034            judges: 0,
8035            present: 0,
8036            quorum: 0,
8037            met_quorum: true,
8038            uncontested: Some("only candidate A produced a change".to_owned()),
8039        });
8040        state.reviews = vec![ReviewRound {
8041            round: 1,
8042            head: "deadbeef".to_owned(),
8043            verified_head: None,
8044            verified_at: None,
8045            reviews: Vec::new(),
8046            e2e: Vec::new(),
8047            fix: None,
8048            blocking: 0,
8049            answered: 0,
8050            expected: 0,
8051            clean: true,
8052            verify_retried: false,
8053            e2e_deferred: false,
8054            e2e_defer_reason: None,
8055            progressed: false,
8056            vote_split: false,
8057            reconsideration: Vec::new(),
8058            verdict: None,
8059        }];
8060        // The point: `gate` has not recorded anything yet.
8061        state.gate = Vec::new();
8062        state.gate_ran = false;
8063        state.status = RunStatus::Gating;
8064
8065        let mut runner = Runner {
8066            state,
8067            roles: ResolvedRoles {
8068                implementers: Vec::new(),
8069                judges: Vec::new(),
8070                reviewers: Vec::new(),
8071                fixer: None,
8072                conductor: conductor(),
8073                implementer_roster: Vec::new(),
8074            },
8075            sem: Arc::new(Semaphore::new(1)),
8076            pause: Pause::new(),
8077            interrupt: Pause::new(),
8078        };
8079
8080        runner.merge().await.expect("merge");
8081
8082        assert!(
8083            runner.state.merge.is_none(),
8084            "an empty gate must never be read as a passing one: {:?}",
8085            runner.state.merge
8086        );
8087    }
8088
8089    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
8090    /// commands configured and `merge.mode` is `none` (a review-only run).
8091    /// `gate` must still record a real attempt — zero commands, vacuously
8092    /// passed — rather than leaving `state.gate` empty in a way `merge`
8093    /// cannot tell apart from "never ran"; otherwise the run reaches
8094    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
8095    #[tokio::test]
8096    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
8097        let tmp = tempfile::tempdir().expect("tempdir");
8098        let repo = tmp.path().join("repo");
8099        std::fs::create_dir_all(&repo).unwrap();
8100        init_repo(&repo);
8101
8102        // Default config: `verify.gate` empty, `merge.mode` is `none`.
8103        let config = Config::default();
8104
8105        let mut state = RunState::new(
8106            repo.clone(),
8107            "main".to_owned(),
8108            "deadbeef".to_owned(),
8109            "task".to_owned(),
8110            config,
8111        );
8112        state.candidates = vec![Candidate {
8113            index: 0,
8114            label: 'A',
8115            agent: "alpha".to_owned(),
8116            branch: "does-not-exist".to_owned(),
8117            worktree: repo.clone(),
8118            summary: String::new(),
8119            stat: String::new(),
8120            files: 0,
8121            commits: 0,
8122            empty: false,
8123            failed: None,
8124            verified_noop: None,
8125            duration_ms: 0,
8126            folded: false,
8127        }];
8128        state.tally = Some(Tally {
8129            first_choice: BTreeMap::from([('A', 1)]),
8130            borda: BTreeMap::new(),
8131            winner: 'A',
8132            rankings: 1,
8133            unanimous_initial: true,
8134            deliberated: false,
8135            changed_votes: 0,
8136            unanimous_final: true,
8137            tie_break: None,
8138            judges: 0,
8139            present: 0,
8140            quorum: 0,
8141            met_quorum: true,
8142            uncontested: Some("only candidate A produced a change".to_owned()),
8143        });
8144        state.reviews = vec![ReviewRound {
8145            round: 1,
8146            head: "deadbeef".to_owned(),
8147            verified_head: None,
8148            verified_at: None,
8149            reviews: Vec::new(),
8150            e2e: Vec::new(),
8151            fix: None,
8152            blocking: 0,
8153            answered: 0,
8154            expected: 0,
8155            clean: true,
8156            verify_retried: false,
8157            e2e_deferred: false,
8158            e2e_defer_reason: None,
8159            progressed: false,
8160            vote_split: false,
8161            reconsideration: Vec::new(),
8162            verdict: None,
8163        }];
8164
8165        let mut runner = Runner {
8166            state,
8167            roles: ResolvedRoles {
8168                implementers: Vec::new(),
8169                judges: Vec::new(),
8170                reviewers: Vec::new(),
8171                fixer: None,
8172                conductor: conductor(),
8173                implementer_roster: Vec::new(),
8174            },
8175            sem: Arc::new(Semaphore::new(1)),
8176            pause: Pause::new(),
8177            interrupt: Pause::new(),
8178        };
8179
8180        runner.gate().await.expect("gate");
8181        assert!(
8182            runner.state.gate_ran,
8183            "zero configured commands is still a real attempt, not an unrun gate"
8184        );
8185        assert!(runner.state.gate.is_empty());
8186        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
8187        assert_ne!(
8188            runner.state.status,
8189            RunStatus::Blocked,
8190            "a gate with nothing to check must not read as failed"
8191        );
8192
8193        runner.merge().await.expect("merge");
8194        assert_eq!(
8195            runner.state.status,
8196            RunStatus::Ready,
8197            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
8198        );
8199    }
8200
8201    /// `Config::cache_dir` is derived from `verify.e2e` as well as
8202    /// `verify.gate` (so the e2e leg and the final gate never build against
8203    /// different directories). With zero `verify.gate` commands but a
8204    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
8205    /// that lease before discovering it had nothing to run - so a repo with
8206    /// no gate commands could come back `resource_blocked` (and therefore
8207    /// still `gate_ran == false`) on nothing but an unrelated run holding the
8208    /// cache, exactly the contention this run's own zero commands could
8209    /// never have touched. `gate` must recognise there is nothing to check
8210    /// before it ever asks for the lease.
8211    #[tokio::test]
8212    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
8213        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8214        let home = crate::run::home();
8215
8216        let tmp = tempfile::tempdir().expect("tempdir");
8217        let repo = tmp.path().join("repo");
8218        std::fs::create_dir_all(&repo).unwrap();
8219        init_repo(&repo);
8220        // Unique to this test, so holding its lease cannot collide with
8221        // another test sharing the same process-wide `home`.
8222        let cache_dir = tmp.path().join("target");
8223
8224        let mut config = Config::default();
8225        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
8226        // `verify.gate` stays empty (the default). Bounded so a regression
8227        // that does start waiting fails the test in seconds, not hangs it.
8228        config.graph.timeout_verify = Some(2);
8229
8230        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8231        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8232            .expect("no io error acquiring directly")
8233        {
8234            crate::cache::AcquireOutcome::Acquired(g) => g,
8235            crate::cache::AcquireOutcome::Busy(b) => {
8236                panic!("expected the direct acquire to win the lease first: {b:?}")
8237            }
8238        };
8239
8240        let mut state = RunState::new(
8241            repo.clone(),
8242            "main".to_owned(),
8243            "deadbeef".to_owned(),
8244            "task".to_owned(),
8245            config,
8246        );
8247        state.candidates = vec![Candidate {
8248            index: 0,
8249            label: 'A',
8250            agent: "alpha".to_owned(),
8251            branch: "does-not-exist".to_owned(),
8252            worktree: repo.clone(),
8253            summary: String::new(),
8254            stat: String::new(),
8255            files: 0,
8256            commits: 0,
8257            empty: false,
8258            failed: None,
8259            verified_noop: None,
8260            duration_ms: 0,
8261            folded: false,
8262        }];
8263        state.tally = Some(Tally {
8264            first_choice: BTreeMap::from([('A', 1)]),
8265            borda: BTreeMap::new(),
8266            winner: 'A',
8267            rankings: 1,
8268            unanimous_initial: true,
8269            deliberated: false,
8270            changed_votes: 0,
8271            unanimous_final: true,
8272            tie_break: None,
8273            judges: 0,
8274            present: 0,
8275            quorum: 0,
8276            met_quorum: true,
8277            uncontested: Some("only candidate A produced a change".to_owned()),
8278        });
8279        state.reviews = vec![ReviewRound {
8280            round: 1,
8281            head: "deadbeef".to_owned(),
8282            verified_head: None,
8283            verified_at: None,
8284            reviews: Vec::new(),
8285            e2e: Vec::new(),
8286            fix: None,
8287            blocking: 0,
8288            answered: 0,
8289            expected: 0,
8290            clean: true,
8291            verify_retried: false,
8292            e2e_deferred: false,
8293            e2e_defer_reason: None,
8294            progressed: false,
8295            vote_split: false,
8296            reconsideration: Vec::new(),
8297            verdict: None,
8298        }];
8299
8300        let mut runner = Runner {
8301            state,
8302            roles: ResolvedRoles {
8303                implementers: Vec::new(),
8304                judges: Vec::new(),
8305                reviewers: Vec::new(),
8306                fixer: None,
8307                conductor: conductor(),
8308                implementer_roster: Vec::new(),
8309            },
8310            sem: Arc::new(Semaphore::new(1)),
8311            pause: Pause::new(),
8312            interrupt: Pause::new(),
8313        };
8314
8315        let started = std::time::Instant::now();
8316        runner.gate().await.expect("gate");
8317        assert!(
8318            started.elapsed() < Duration::from_secs(1),
8319            "a gate with nothing to run must never wait on a lease it never needed"
8320        );
8321        assert!(
8322            runner.state.gate_ran,
8323            "zero commands is still a real, immediate attempt"
8324        );
8325        assert!(runner.state.gate.is_empty());
8326        assert_ne!(
8327            runner.state.status,
8328            RunStatus::Blocked,
8329            "must not read as resource-blocked on a lease it never asked for"
8330        );
8331    }
8332
8333    /// The addendum's second gap: a `verify.gate` command running for real
8334    /// wall-clock time had nothing at all to show for it in `active` before
8335    /// `run_commands` learned to record it — a run could sit in `Gating` for
8336    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
8337    /// what was actually happening. Proven with a genuinely still-running
8338    /// command, not just a before/after check on the final state: a poller
8339    /// task reads the same `run.json` `gate()` is writing, the same way the
8340    /// phone or `magi show` would, while the shell command is still blocked
8341    /// on its own release marker.
8342    #[tokio::test]
8343    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
8344        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8345
8346        let tmp = tempfile::tempdir().expect("tempdir");
8347        let repo = tmp.path().join("repo");
8348        std::fs::create_dir_all(&repo).unwrap();
8349        init_repo(&repo);
8350
8351        let mut config = Config::default();
8352        config.verify.gate = vec![
8353            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
8354             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
8355                .to_owned(),
8356        ];
8357
8358        let mut state = RunState::new(
8359            repo.clone(),
8360            "main".to_owned(),
8361            "deadbeef".to_owned(),
8362            "task".to_owned(),
8363            config,
8364        );
8365        let run_id = state.id.clone();
8366        state.candidates = vec![Candidate {
8367            index: 0,
8368            label: 'A',
8369            agent: "alpha".to_owned(),
8370            branch: "does-not-exist".to_owned(),
8371            worktree: repo.clone(),
8372            summary: String::new(),
8373            stat: String::new(),
8374            files: 0,
8375            commits: 0,
8376            empty: false,
8377            failed: None,
8378            verified_noop: None,
8379            duration_ms: 0,
8380            folded: false,
8381        }];
8382        state.tally = Some(Tally {
8383            first_choice: BTreeMap::from([('A', 1)]),
8384            borda: BTreeMap::new(),
8385            winner: 'A',
8386            rankings: 1,
8387            unanimous_initial: true,
8388            deliberated: false,
8389            changed_votes: 0,
8390            unanimous_final: true,
8391            tie_break: None,
8392            judges: 0,
8393            present: 0,
8394            quorum: 0,
8395            met_quorum: true,
8396            uncontested: Some("only candidate A produced a change".to_owned()),
8397        });
8398        state.reviews = vec![ReviewRound {
8399            round: 1,
8400            head: "deadbeef".to_owned(),
8401            verified_head: None,
8402            verified_at: None,
8403            reviews: Vec::new(),
8404            e2e: Vec::new(),
8405            fix: None,
8406            blocking: 0,
8407            answered: 0,
8408            expected: 0,
8409            clean: true,
8410            verify_retried: false,
8411            e2e_deferred: false,
8412            e2e_defer_reason: None,
8413            progressed: false,
8414            vote_split: false,
8415            reconsideration: Vec::new(),
8416            verdict: None,
8417        }];
8418
8419        let mut runner = Runner {
8420            state,
8421            roles: ResolvedRoles {
8422                implementers: Vec::new(),
8423                judges: Vec::new(),
8424                reviewers: Vec::new(),
8425                fixer: None,
8426                conductor: conductor(),
8427                implementer_roster: Vec::new(),
8428            },
8429            sem: Arc::new(Semaphore::new(1)),
8430            pause: Pause::new(),
8431            interrupt: Pause::new(),
8432        };
8433
8434        let started_marker = repo.join("started.marker");
8435        let release_marker = repo.join("release.marker");
8436        let poller = tokio::spawn(async move {
8437            // Bounded so a regression that never records the task entry
8438            // fails this test in seconds instead of hanging the suite —
8439            // the same shape `a_park_requested_while_a_seat_is_mid_call_
8440            // does_not_cut_it_short` uses for the same reason.
8441            for _ in 0..100 {
8442                if started_marker.exists()
8443                    && let Ok(s) = crate::run::RunState::load(&run_id)
8444                    && let Some(a) = s.active.get("gate")
8445                {
8446                    std::fs::write(&release_marker, b"go").expect("release marker");
8447                    return Some(a.clone());
8448                }
8449                tokio::time::sleep(Duration::from_millis(50)).await;
8450            }
8451            None
8452        });
8453
8454        runner.gate().await.expect("gate");
8455        let captured = poller.await.expect("poller task");
8456        let captured = captured.expect(
8457            "the poller never saw a `gate` task entry in run.json while the command was \
8458             still blocked on its own release marker",
8459        );
8460
8461        assert_eq!(captured.task.as_deref(), Some("gate"));
8462        assert_eq!(captured.node, "gate");
8463        assert_eq!(captured.index, Some(1));
8464        assert_eq!(captured.total, Some(1));
8465        assert!(
8466            captured
8467                .command
8468                .as_deref()
8469                .is_some_and(|c| c.contains("started.marker")),
8470            "{captured:?}"
8471        );
8472
8473        assert!(
8474            runner.state.active.is_empty(),
8475            "the entry must be cleared once the command actually finished: {:?}",
8476            runner.state.active
8477        );
8478        assert!(runner.state.gate_ran);
8479        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
8480    }
8481
8482    /// The shape the incident this whole fix responds to actually had: the
8483    /// round budget spent, the last round's own e2e blocked on the shared
8484    /// build cache (held here by a live pid — this test process — exactly
8485    /// `cache`'s own unit tests' pattern for "another owner, still alive"
8486    /// without forking a process). `stop_reviewing` must retry it — not
8487    /// silently leave the round looking untouched (the catch-up-only half of
8488    /// the bug), and not read the contention as a red `e2e` and block the
8489    /// run on it (the other half). Called directly, the same way
8490    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
8491    /// above exercises `gate`, so this never needs a real cargo build to
8492    /// reach: the lease is never released, so `with_cache_lease` never gets
8493    /// past acquiring it into anything that would need a real workspace.
8494    #[tokio::test]
8495    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
8496        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8497        let home = crate::run::home();
8498
8499        let tmp = tempfile::tempdir().expect("tempdir");
8500        let repo = tmp.path().join("repo");
8501        std::fs::create_dir_all(&repo).unwrap();
8502        init_repo(&repo);
8503        let head = crate::git::rev_parse(&repo, "HEAD")
8504            .await
8505            .expect("rev-parse");
8506        // Unique to this test, so holding its lease cannot collide with
8507        // another test sharing the same process-wide `home`.
8508        let cache_dir = tmp.path().join("target");
8509
8510        let mut config = Config::default();
8511        config.verify.e2e = vec![format!(
8512            "CARGO_TARGET_DIR='{}' test -f README.md",
8513            cache_dir.display()
8514        )];
8515        config.graph.review_rounds = 1;
8516        // Bounded so a regression that does start waiting fails the test in
8517        // seconds, not hangs it.
8518        config.graph.timeout_verify = Some(2);
8519
8520        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8521        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8522            .expect("no io error acquiring directly")
8523        {
8524            crate::cache::AcquireOutcome::Acquired(g) => g,
8525            crate::cache::AcquireOutcome::Busy(b) => {
8526                panic!("expected the direct acquire to win the lease first: {b:?}")
8527            }
8528        };
8529
8530        let mut state = RunState::new(
8531            repo.clone(),
8532            "main".to_owned(),
8533            head.clone(),
8534            "task".to_owned(),
8535            config,
8536        );
8537        state.candidates = vec![Candidate {
8538            index: 0,
8539            label: 'A',
8540            agent: "alpha".to_owned(),
8541            branch: "does-not-exist".to_owned(),
8542            worktree: repo.clone(),
8543            summary: String::new(),
8544            stat: String::new(),
8545            files: 0,
8546            commits: 0,
8547            empty: false,
8548            failed: None,
8549            verified_noop: None,
8550            duration_ms: 0,
8551            folded: false,
8552        }];
8553        state.tally = Some(Tally {
8554            first_choice: BTreeMap::from([('A', 1)]),
8555            borda: BTreeMap::new(),
8556            winner: 'A',
8557            rankings: 1,
8558            unanimous_initial: true,
8559            deliberated: false,
8560            changed_votes: 0,
8561            unanimous_final: true,
8562            tie_break: None,
8563            judges: 0,
8564            present: 0,
8565            quorum: 0,
8566            met_quorum: true,
8567            uncontested: Some("only candidate A produced a change".to_owned()),
8568        });
8569        // The round budget's last round, deferred: `needs_catchup_run`'s
8570        // other trigger. `stop_reviewing`'s retry machinery must treat this
8571        // exactly like a resource-blocked attempt once it actually runs.
8572        state.reviews = vec![ReviewRound {
8573            round: 1,
8574            head: head.clone(),
8575            verified_head: None,
8576            verified_at: None,
8577            reviews: Vec::new(),
8578            e2e: Vec::new(),
8579            fix: None,
8580            blocking: 1,
8581            answered: 1,
8582            expected: 1,
8583            clean: false,
8584            verify_retried: false,
8585            e2e_deferred: true,
8586            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
8587            progressed: false,
8588            vote_split: false,
8589            reconsideration: Vec::new(),
8590            verdict: None,
8591        }];
8592
8593        let mut runner = Runner {
8594            state,
8595            roles: ResolvedRoles {
8596                implementers: Vec::new(),
8597                judges: Vec::new(),
8598                reviewers: Vec::new(),
8599                fixer: None,
8600                conductor: conductor(),
8601                implementer_roster: Vec::new(),
8602            },
8603            sem: Arc::new(Semaphore::new(1)),
8604            pause: Pause::new(),
8605            interrupt: Pause::new(),
8606        };
8607
8608        let shell = runner.state.config.shell();
8609        runner
8610            .stop_reviewing("round budget spent", &shell, &repo)
8611            .await
8612            .expect("stop_reviewing");
8613
8614        let last = runner.state.reviews.last().expect("round record");
8615        assert_eq!(
8616            last.e2e_status(),
8617            E2eStatus::ResourceBlocked,
8618            "the shared cache is still held; the attempt must read as blocked, not deferred or \
8619             failed: {last:?}"
8620        );
8621        assert_eq!(
8622            last.verified_head.as_deref(),
8623            Some(head.as_str()),
8624            "which commit this attempt targeted is known even though nothing finished checking \
8625             it"
8626        );
8627        let first_attempt_at = last
8628            .verified_at
8629            .expect("when this attempt ran is known too");
8630        assert_ne!(
8631            runner.state.status,
8632            RunStatus::Blocked,
8633            "contention is evidence about the machine, not the patch — it must not settle the \
8634             run as blocked: {:?}",
8635            runner.state.status
8636        );
8637        assert!(
8638            !runner
8639                .state
8640                .events
8641                .iter()
8642                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
8643            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
8644            runner.state.events
8645        );
8646
8647        // The cache is still held: a later reentry must retry the same
8648        // round's verification again — not leave it looking exactly as
8649        // untouched as the first blocked attempt, which is indistinguishable
8650        // from never having tried again at all.
8651        runner
8652            .stop_reviewing("round budget spent", &shell, &repo)
8653            .await
8654            .expect("stop_reviewing retry");
8655        assert_eq!(
8656            runner.state.reviews.len(),
8657            1,
8658            "no new round was started: {:?}",
8659            runner.state.reviews
8660        );
8661        let last = runner.state.reviews.last().expect("round record");
8662        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
8663        assert!(
8664            last.verified_at.expect("still known") > first_attempt_at,
8665            "a second reentry must be a fresh attempt, not a stale copy of the first"
8666        );
8667        assert_ne!(runner.state.status, RunStatus::Blocked);
8668
8669        held.release();
8670    }
8671
8672    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
8673    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
8674    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
8675    /// own top-of-function fast path (`review_conclusion`) correctly reads
8676    /// this shape as `None` rather than guessing `Blocked`, and the loop's
8677    /// own `for` range is empty once the round budget is spent, so
8678    /// `review_loop` must retry the check itself rather than silently doing
8679    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
8680    /// above exercises directly, but through `review_loop`'s own entry point
8681    /// this time, proving the wiring between the two rather than just the
8682    /// retry logic in isolation.
8683    #[tokio::test]
8684    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
8685        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8686        let home = crate::run::home();
8687
8688        let tmp = tempfile::tempdir().expect("tempdir");
8689        let repo = tmp.path().join("repo");
8690        std::fs::create_dir_all(&repo).unwrap();
8691        init_repo(&repo);
8692        let head = crate::git::rev_parse(&repo, "HEAD")
8693            .await
8694            .expect("rev-parse");
8695        let cache_dir = tmp.path().join("target");
8696
8697        let mut config = Config::default();
8698        config.verify.e2e = vec![format!(
8699            "CARGO_TARGET_DIR='{}' test -f README.md",
8700            cache_dir.display()
8701        )];
8702        config.graph.review_rounds = 1;
8703        config.graph.timeout_verify = Some(2);
8704
8705        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8706        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8707            .expect("no io error acquiring directly")
8708        {
8709            crate::cache::AcquireOutcome::Acquired(g) => g,
8710            crate::cache::AcquireOutcome::Busy(b) => {
8711                panic!("expected the direct acquire to win the lease first: {b:?}")
8712            }
8713        };
8714
8715        let mut state = RunState::new(
8716            repo.clone(),
8717            "main".to_owned(),
8718            head.clone(),
8719            "task".to_owned(),
8720            config,
8721        );
8722        state.candidates = vec![Candidate {
8723            index: 0,
8724            label: 'A',
8725            agent: "alpha".to_owned(),
8726            branch: "does-not-exist".to_owned(),
8727            worktree: repo.clone(),
8728            summary: String::new(),
8729            stat: String::new(),
8730            files: 0,
8731            commits: 0,
8732            empty: false,
8733            failed: None,
8734            verified_noop: None,
8735            duration_ms: 0,
8736            folded: false,
8737        }];
8738        state.tally = Some(Tally {
8739            first_choice: BTreeMap::from([('A', 1)]),
8740            borda: BTreeMap::new(),
8741            winner: 'A',
8742            rankings: 1,
8743            unanimous_initial: true,
8744            deliberated: false,
8745            changed_votes: 0,
8746            unanimous_final: true,
8747            tie_break: None,
8748            judges: 0,
8749            present: 0,
8750            quorum: 0,
8751            met_quorum: true,
8752            uncontested: Some("only candidate A produced a change".to_owned()),
8753        });
8754        // The exact shape a prior process's `stop_reviewing` would have left
8755        // on disk: the round budget's last round, a real attempt already
8756        // made and already resource-blocked.
8757        state.reviews = vec![ReviewRound {
8758            round: 1,
8759            head: head.clone(),
8760            verified_head: Some(head.clone()),
8761            verified_at: Some(jiff::Timestamp::now()),
8762            reviews: Vec::new(),
8763            e2e: vec![CommandOutcome {
8764                command: format!(
8765                    "CARGO_TARGET_DIR='{}' test -f README.md",
8766                    cache_dir.display()
8767                ),
8768                code: None,
8769                output_tail: "waiting for the shared build cache".to_owned(),
8770                duration_ms: 0,
8771                resource_blocked: true,
8772            }],
8773            fix: None,
8774            blocking: 1,
8775            answered: 1,
8776            expected: 1,
8777            clean: false,
8778            verify_retried: false,
8779            e2e_deferred: false,
8780            e2e_defer_reason: None,
8781            progressed: false,
8782            vote_split: false,
8783            reconsideration: Vec::new(),
8784            verdict: None,
8785        }];
8786
8787        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
8788        let mut runner = Runner {
8789            state,
8790            roles: ResolvedRoles {
8791                implementers: Vec::new(),
8792                judges: Vec::new(),
8793                reviewers: Vec::new(),
8794                fixer: None,
8795                conductor: conductor(),
8796                implementer_roster: Vec::new(),
8797            },
8798            sem: Arc::new(Semaphore::new(1)),
8799            pause: Pause::new(),
8800            interrupt: Pause::new(),
8801        };
8802
8803        // The lease is still held throughout, so this reentry's own retry is
8804        // also contended — proving `review_loop` actually tried again (not
8805        // that it happened to succeed) is what the timestamp comparison
8806        // below is for.
8807        runner.review_loop().await.expect("review_loop");
8808
8809        assert_eq!(
8810            runner.state.reviews.len(),
8811            1,
8812            "no new round was started on top of the unresolved one: {:?}",
8813            runner.state.reviews
8814        );
8815        let last = &runner.state.reviews[0];
8816        assert_eq!(
8817            last.e2e_status(),
8818            E2eStatus::ResourceBlocked,
8819            "still contended: {last:?}"
8820        );
8821        assert!(
8822            last.verified_at.expect("still known") > first_attempt_at,
8823            "review_loop must have actually retried the check, not left it exactly as found"
8824        );
8825        assert_ne!(
8826            runner.state.status,
8827            RunStatus::Blocked,
8828            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
8829            runner.state.status
8830        );
8831
8832        held.release();
8833    }
8834
8835    #[tokio::test]
8836    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
8837        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8838        let tmp = tempfile::tempdir().expect("tempdir");
8839        let repo = tmp.path().join("repo");
8840        std::fs::create_dir_all(&repo).unwrap();
8841        init_repo(&repo);
8842
8843        let mut config = Config::default();
8844        config.merge.mode = MergeMode::Pr;
8845        config.graph.land = true;
8846        config.graph.land_approval = false;
8847
8848        let mut state = RunState::new(
8849            repo.clone(),
8850            "main".to_owned(),
8851            "deadbeef".to_owned(),
8852            "task".to_owned(),
8853            config,
8854        );
8855        state.candidates = vec![Candidate {
8856            index: 0,
8857            label: 'A',
8858            agent: "alpha".to_owned(),
8859            branch: "does-not-exist".to_owned(),
8860            worktree: repo.clone(),
8861            summary: String::new(),
8862            stat: String::new(),
8863            files: 0,
8864            commits: 0,
8865            empty: false,
8866            failed: None,
8867            verified_noop: None,
8868            duration_ms: 0,
8869            folded: false,
8870        }];
8871        state.tally = Some(Tally {
8872            first_choice: BTreeMap::from([('A', 1)]),
8873            borda: BTreeMap::new(),
8874            winner: 'A',
8875            rankings: 1,
8876            unanimous_initial: true,
8877            deliberated: false,
8878            changed_votes: 0,
8879            unanimous_final: true,
8880            tie_break: None,
8881            judges: 0,
8882            present: 0,
8883            quorum: 0,
8884            met_quorum: true,
8885            uncontested: Some("only candidate A produced a change".to_owned()),
8886        });
8887        state.reviews = vec![ReviewRound {
8888            round: 1,
8889            head: "deadbeef".to_owned(),
8890            verified_head: None,
8891            verified_at: None,
8892            reviews: Vec::new(),
8893            e2e: Vec::new(),
8894            fix: None,
8895            blocking: 0,
8896            answered: 0,
8897            expected: 0,
8898            clean: true,
8899            verify_retried: false,
8900            e2e_deferred: false,
8901            e2e_defer_reason: None,
8902            progressed: false,
8903            vote_split: false,
8904            reconsideration: Vec::new(),
8905            verdict: None,
8906        }];
8907        state.gate = vec![CommandOutcome {
8908            command: "test".to_owned(),
8909            code: Some(0),
8910            output_tail: String::new(),
8911            duration_ms: 0,
8912            resource_blocked: false,
8913        }];
8914        state.gate_ran = true;
8915        // A first pass through `merge` already pushed and opened this pull
8916        // request; `status` is `Landing` because a previous call into `land`
8917        // parked or was interrupted before it reached a terminal outcome.
8918        state.status = RunStatus::Landing;
8919        state.merge = Some(MergeOutcome {
8920            mode: MergeMode::Pr,
8921            ok: true,
8922            detail: "https://example.invalid/x/y/pull/1".to_owned(),
8923        });
8924
8925        // The Landing-resume shortcut calls `run_land` directly rather than
8926        // through `merge`, which is exactly the call site that used to skip
8927        // `settle_questions` - see the fixture below.
8928        ask_test_home();
8929        let store = ask::Questions::open();
8930        let q = ask_open_question(&store, &state.id);
8931
8932        let mut runner = Runner {
8933            state,
8934            roles: ResolvedRoles {
8935                implementers: Vec::new(),
8936                judges: Vec::new(),
8937                reviewers: Vec::new(),
8938                fixer: None,
8939                conductor: conductor(),
8940                implementer_roster: Vec::new(),
8941            },
8942            sem: Arc::new(Semaphore::new(1)),
8943            pause: Pause::new(),
8944            interrupt: Pause::new(),
8945        };
8946
8947        // `execute`, not `merge` directly: the Landing-resume shortcut lives
8948        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
8949        // exactly because `review_loop` would otherwise clobber the marker
8950        // first.
8951        runner.execute().await.expect("execute");
8952
8953        assert_eq!(
8954            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
8955            Some("https://example.invalid/x/y/pull/1"),
8956            "reentry must not push again or open a second pull request over the \
8957             one `land` is already watching"
8958        );
8959        assert_ne!(
8960            runner.state.status,
8961            RunStatus::Landing,
8962            "land could not actually reach the fake pull request, so it must \
8963             have given up rather than left the run silently parked forever"
8964        );
8965        // `land` could not reach the fake pull request, so it gave up into
8966        // `Blocked` - still resumable, so the question must not have been
8967        // swept just because this branch now also calls `settle_questions`.
8968        assert_eq!(runner.state.status, RunStatus::Blocked);
8969        assert!(
8970            store.get(&q.id).unwrap().status.open(),
8971            "Blocked is still alive; settle_questions must have been a no-op here"
8972        );
8973    }
8974
8975    fn state_with_round(round: ReviewRound) -> RunState {
8976        let mut s = RunState::new(
8977            PathBuf::from("/repo"),
8978            "main".to_owned(),
8979            "abc1234".to_owned(),
8980            "add retries".to_owned(),
8981            Config::default(),
8982        );
8983        s.reviews = vec![round];
8984        s
8985    }
8986
8987    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
8988        crate::verdict::Finding {
8989            id: id.to_owned(),
8990            severity,
8991            file: None,
8992            line: None,
8993            title: title.to_owned(),
8994            detail: String::new(),
8995        }
8996    }
8997
8998    #[test]
8999    fn pr_body_names_open_findings_and_declined_ones() {
9000        let round = ReviewRound {
9001            round: 2,
9002            head: "deadbee".to_owned(),
9003            verified_head: None,
9004            verified_at: None,
9005            reviews: vec![ReviewRecord {
9006                attempts: 0,
9007                reviewer: 1,
9008                agent: "alpha".to_owned(),
9009                summary: String::new(),
9010                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
9011                vote: None,
9012                failed: None,
9013                duration_ms: 0,
9014            }],
9015            e2e: vec![CommandOutcome {
9016                command: "cargo test".to_owned(),
9017                code: Some(0),
9018                output_tail: String::new(),
9019                duration_ms: 0,
9020                resource_blocked: false,
9021            }],
9022            verify_retried: false,
9023            e2e_deferred: false,
9024            e2e_defer_reason: None,
9025            fix: Some(FixRecord {
9026                agent: "alpha".to_owned(),
9027                addressed: Vec::new(),
9028                rejected: vec![crate::verdict::Rejection {
9029                    id: "R1-1-1".to_owned(),
9030                    why: "not reachable from any caller".to_owned(),
9031                }],
9032                notes: String::new(),
9033                committed: true,
9034                failed: None,
9035                duration_ms: 0,
9036                continuation: None,
9037            }),
9038            blocking: 0,
9039            answered: 1,
9040            expected: 1,
9041            clean: false,
9042            progressed: true,
9043            vote_split: false,
9044            reconsideration: Vec::new(),
9045            verdict: None,
9046        };
9047        let state = state_with_round(round);
9048        let body = pr_message(&state, 'A').body;
9049
9050        assert!(body.contains("add retries"), "the task must still be there");
9051        assert!(body.contains("R2-1-1"), "{body}");
9052        assert!(body.contains("unused import"), "{body}");
9053        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
9054        assert!(
9055            body.contains("not reachable from any caller"),
9056            "the reason it was declined: {body}"
9057        );
9058    }
9059
9060    #[test]
9061    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
9062        let round = ReviewRound {
9063            round: 1,
9064            head: "deadbee".to_owned(),
9065            verified_head: None,
9066            verified_at: None,
9067            reviews: vec![ReviewRecord {
9068                attempts: 0,
9069                reviewer: 1,
9070                agent: "alpha".to_owned(),
9071                summary: String::new(),
9072                findings: Vec::new(),
9073                vote: None,
9074                failed: None,
9075                duration_ms: 0,
9076            }],
9077            e2e: Vec::new(),
9078            verify_retried: false,
9079            e2e_deferred: false,
9080            e2e_defer_reason: None,
9081            fix: None,
9082            blocking: 0,
9083            answered: 1,
9084            expected: 1,
9085            clean: true,
9086            progressed: false,
9087            vote_split: false,
9088            reconsideration: Vec::new(),
9089            verdict: None,
9090        };
9091        let state = state_with_round(round);
9092        let body = pr_message(&state, 'A').body;
9093        assert!(!body.contains("Open review findings"), "{body}");
9094        assert!(!body.contains("Declined"), "{body}");
9095    }
9096
9097    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
9098        let mut state = RunState::new(
9099            PathBuf::from("/repo"),
9100            "main".to_owned(),
9101            "abc1234".to_owned(),
9102            instruction.to_owned(),
9103            Config::default(),
9104        );
9105        state.candidates.push(Candidate {
9106            index: 0,
9107            label: 'A',
9108            agent: "alpha".to_owned(),
9109            branch: "magi/x/A".to_owned(),
9110            worktree: PathBuf::from("/wt"),
9111            summary: summary.to_owned(),
9112            stat: String::new(),
9113            files: 1,
9114            commits: 1,
9115            empty: false,
9116            failed: None,
9117            verified_noop: None,
9118            folded: false,
9119            duration_ms: 0,
9120        });
9121        state
9122    }
9123
9124    #[test]
9125    fn pr_message_describes_the_change_not_the_task() {
9126        let state = state_with_summary(
9127            "今回やってほしいこと: results projector を直す",
9128            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
9129        );
9130        let m = pr_message(&state, 'A');
9131        assert_eq!(m.title, "fix(web): batch the runs list reads");
9132        assert!(
9133            m.body.starts_with("## Summary\n\n- reads run.json once"),
9134            "{}",
9135            m.body
9136        );
9137        assert!(!m.body.contains("TITLE:"), "{}", m.body);
9138        let task_at = m.body.find("今回やってほしいこと").unwrap();
9139        let details_at = m.body.find("<details>").unwrap();
9140        assert!(
9141            details_at < task_at,
9142            "the task lives inside <details>: {}",
9143            m.body
9144        );
9145        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
9146        assert!(m.body.contains("magi:candidate-a"));
9147    }
9148
9149    #[test]
9150    fn pr_message_falls_back_to_the_task_without_a_title_line() {
9151        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
9152        let m = pr_message(&state, 'A');
9153        assert_eq!(m.title, "add retries");
9154        assert!(
9155            m.body.contains("## Summary\n\n- did some things"),
9156            "{}",
9157            m.body
9158        );
9159
9160        let none = RunState::new(
9161            PathBuf::from("/repo"),
9162            "main".to_owned(),
9163            "abc1234".to_owned(),
9164            "add retries".to_owned(),
9165            Config::default(),
9166        );
9167        let m = pr_message(&none, 'A');
9168        assert_eq!(m.title, "add retries");
9169        assert!(!m.body.contains("## Summary"), "{}", m.body);
9170    }
9171
9172    #[test]
9173    fn pr_message_refuses_the_candidate_commit_subject() {
9174        for bad in [
9175            "TITLE: magi: candidate A (uncommitted work)",
9176            "TITLE: chore: stuff (uncommitted work)",
9177            "TITLE:   ",
9178        ] {
9179            let state = state_with_summary("add retries", bad);
9180            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
9181        }
9182    }
9183
9184    #[test]
9185    fn pr_message_bounds_a_very_long_task_and_title() {
9186        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
9187        let state = state_with_summary(&long, "- nothing");
9188        let m = pr_message(&state, 'A');
9189        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
9190        assert!(!m.title.contains('\n'));
9191
9192        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
9193        let m = pr_message(&state, 'A');
9194        assert!(m.title.starts_with("feat: "));
9195        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
9196        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
9197    }
9198
9199    #[test]
9200    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
9201        // What magi itself writes stays English under any configured language,
9202        // so a future localisation of these headings fails here. (The agents'
9203        // own text is held to English by the prompt only; magi cannot check it.)
9204        let mut state = state_with_summary(
9205            "add retries",
9206            "TITLE: fix(web): batch reads\n- reads run.json once",
9207        );
9208        state.config.graph.language = "ja".to_owned();
9209        let m = pr_message(&state, 'A');
9210        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
9211
9212        // The task is the operator's own text: it goes in untouched, and the
9213        // fallback title (no summary) may be in its language too.
9214        let task = "今回やってほしいこと: results projector を直す";
9215        let mut state = state_with_summary(task, "- no title line");
9216        state.config.graph.language = "ja".to_owned();
9217        let m = pr_message(&state, 'A');
9218        assert_eq!(
9219            m.title,
9220            format!("chore: land candidate A of run {}", state.id)
9221        );
9222        assert!(
9223            m.body.contains(&format!(
9224                "<summary>Original task</summary>\n\n{task}\n\n</details>"
9225            )),
9226            "{}",
9227            m.body
9228        );
9229    }
9230
9231    #[test]
9232    fn pr_message_scrubs_home_paths_and_addresses() {
9233        let state = state_with_summary(
9234            "fix it in /Users/someone/src/x",
9235            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
9236        );
9237        let m = pr_message(&state, 'A');
9238        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
9239            assert!(!m.body.contains(leak), "{}", m.body);
9240        }
9241        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
9242    }
9243
9244    #[test]
9245    fn pr_message_survives_a_task_that_closes_details() {
9246        let state = state_with_summary("a </details> b", "TITLE: fix: x");
9247        let m = pr_message(&state, 'A');
9248        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
9249    }
9250
9251    #[test]
9252    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
9253        let cmd = manual_merge_command(
9254            MergeStyle::Squash,
9255            Path::new("/repo"),
9256            "b",
9257            "fix: \"quoted\" $(x) `y`\n\nbody",
9258        );
9259        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
9260    }
9261
9262    #[test]
9263    fn manual_merge_command_matches_the_configured_style() {
9264        let repo = Path::new("/repo");
9265        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
9266
9267        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
9268        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
9269
9270        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
9271        assert_eq!(
9272            squash,
9273            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
9274             \"Merge magi run 0832 (candidate A)\""
9275        );
9276
9277        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
9278        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
9279    }
9280
9281    #[test]
9282    fn a_nudge_gets_a_quarter_of_the_budget() {
9283        // The judge and implement budgets magi ships with.
9284        assert_eq!(retry_budget(secs(1200), true), secs(300));
9285        assert_eq!(retry_budget(secs(3600), true), secs(900));
9286    }
9287
9288    #[test]
9289    fn a_resent_prompt_keeps_the_whole_budget() {
9290        // The seat kept no context, so the retry is the original job again and
9291        // shortening it would only guarantee a second failure.
9292        assert_eq!(retry_budget(secs(1200), false), secs(1200));
9293        assert_eq!(retry_budget(secs(60), false), secs(60));
9294    }
9295
9296    #[test]
9297    fn the_floor_never_exceeds_the_original_budget() {
9298        // A short configured timeout must not be *raised* by the floor: the
9299        // operator asked for a bound, and a retry may not outlast the attempt
9300        // it is retrying.
9301        assert_eq!(retry_budget(secs(60), true), secs(60));
9302        assert_eq!(retry_budget(secs(480), true), secs(120));
9303        assert_eq!(retry_budget(secs(0), true), secs(0));
9304    }
9305
9306    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
9307        agent::CommandEvidence {
9308            id: "item1".to_owned(),
9309            description: "cargo test".to_owned(),
9310            exit_code,
9311            result_summary: String::new(),
9312            source: "codex".to_owned(),
9313        }
9314    }
9315
9316    #[test]
9317    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
9318        // No evidence is not the same fact as unconfirmed evidence: a
9319        // backend with no adapter, or a reply that ran no commands at all,
9320        // must not be misread as carrying a dangling job.
9321        assert!(!has_unconfirmed_command(&[]));
9322    }
9323
9324    #[test]
9325    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
9326        // Deliberately not a check on the exit code's *value*: a fixer
9327        // legitimately runs something that fails mid-iteration before it
9328        // succeeds, and that must never by itself reopen a valid report.
9329        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
9330        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
9331        assert!(!has_unconfirmed_command(&[
9332            evidence(Some(0)),
9333            evidence(Some(101))
9334        ]));
9335    }
9336
9337    #[test]
9338    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
9339        assert!(has_unconfirmed_command(&[
9340            evidence(Some(0)),
9341            evidence(None)
9342        ]));
9343    }
9344
9345    #[test]
9346    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
9347        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9348        assert_eq!(
9349            verified_noop_claim(true, &[], text).as_deref(),
9350            Some("already fixed by b32cfc4, on main.")
9351        );
9352    }
9353
9354    #[test]
9355    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
9356        // A timeout or a bad exit code reads as the ordinary loss it is,
9357        // whatever the reply's own prose claims.
9358        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9359        assert!(verified_noop_claim(false, &[], text).is_none());
9360    }
9361
9362    #[test]
9363    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
9364        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9365        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
9366        // A confirmed command alongside the marker is fine.
9367        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
9368    }
9369
9370    #[test]
9371    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
9372        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
9373    }
9374
9375    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
9376    /// pair, in order, labelled A, B, C, ...
9377    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
9378        runner.state.candidates = shape
9379            .iter()
9380            .enumerate()
9381            .map(|(i, &(empty, verified))| Candidate {
9382                index: i,
9383                label: (b'A' + i as u8) as char,
9384                agent: "sonnet".to_owned(),
9385                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
9386                worktree: PathBuf::from(format!("/wt/{i}")),
9387                summary: String::new(),
9388                stat: String::new(),
9389                files: 0,
9390                commits: 0,
9391                empty,
9392                failed: None,
9393                verified_noop: verified.map(str::to_owned),
9394                duration_ms: 0,
9395                folded: false,
9396            })
9397            .collect();
9398    }
9399
9400    #[test]
9401    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
9402        ask_test_home();
9403        let mut runner = runner_at(RunStatus::Implementing);
9404        set_candidates(
9405            &mut runner,
9406            &[
9407                (true, Some("already on main at b32cfc4")),
9408                (true, Some("same fix, see the existing test")),
9409            ],
9410        );
9411
9412        runner
9413            .after_implement()
9414            .expect("a verified no-op is not an error");
9415
9416        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
9417    }
9418
9419    #[test]
9420    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
9421        ask_test_home();
9422        let mut runner = runner_at(RunStatus::Implementing);
9423        // Candidate A declares a verified no-op; candidate B simply wrote
9424        // nothing and said nothing about why. One candidate's claim is not
9425        // the whole run's agreement.
9426        set_candidates(
9427            &mut runner,
9428            &[(true, Some("already on main at b32cfc4")), (true, None)],
9429        );
9430
9431        let err = runner
9432            .after_implement()
9433            .expect_err("an unverified empty candidate must still fail the run");
9434
9435        assert!(
9436            err.to_string().contains("no candidate produced a change"),
9437            "{err}"
9438        );
9439        assert_eq!(runner.state.status, RunStatus::Failed);
9440    }
9441
9442    #[test]
9443    fn after_implement_still_fails_an_ordinary_all_empty_run() {
9444        ask_test_home();
9445        let mut runner = runner_at(RunStatus::Implementing);
9446        set_candidates(&mut runner, &[(true, None), (true, None)]);
9447
9448        let err = runner
9449            .after_implement()
9450            .expect_err("no candidate declared anything; this is an ordinary failure");
9451
9452        assert!(
9453            err.to_string().contains("no candidate produced a change"),
9454            "{err}"
9455        );
9456        assert_eq!(runner.state.status, RunStatus::Failed);
9457    }
9458}