Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::run::{
46    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
47    DeliberationRound, DeliberationTurn, E2eStatus, FixRecord, GateFixRecord, JobRecord, JobStatus,
48    Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, QuotaLoss,
49    ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord, tail,
50    write_artifact,
51};
52use crate::verdict::{
53    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
54    ReviewVote, Severity,
55};
56
57/// How much verification output is kept and fed back to the fixer.
58const OUTPUT_TAIL: usize = 8_000;
59
60/// Bytes of a failing command's output kept in an event, so the reason a run
61/// stopped is readable from the report without opening `run.json`.
62const EVENT_OUTPUT_TAIL: usize = 2_000;
63
64/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
65/// command's pid before releasing the build cache's lease.
66const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
67
68/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
69/// command's pid to actually exit before giving up and releasing anyway.
70///
71/// A timeout means the process was asked to die (`kill_on_drop`,
72/// `start_kill`), not that it already has — on Windows in particular that can
73/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
74/// the instant the command returns would let the very next acquirer (this
75/// run's own next round, another run's verification, the janitor's prune)
76/// start touching the same directory while it might still be writing to it,
77/// so this polls the actual pid — real confirmation, not a fixed guess —
78/// until it is gone or this ceiling is reached. It is still not full
79/// process-tree reaping: a grandchild the timed-out process spawned and that
80/// outlives it independently is invisible to a pid check, and continuing to
81/// observe and collect *that* stays a different piece of work with its own
82/// owner. Set generously because the common case returns early the moment
83/// the pid is confirmed gone, not because every timeout pays this in full.
84const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
85
86/// Consecutive review rounds with no tree progress (see
87/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
88/// instead of spending the rest of the round budget.
89///
90/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
91/// legitimately finds nothing left to change (its previous round's fix already
92/// covered it, and this round's reviewers re-raised only nits) looks the same
93/// as one that is spinning, for exactly one round. Two in a row is where the
94/// two stop being distinguishable, and a review round on this workload has
95/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
96/// third attempt at a tree that has not moved twice running is pure cost.
97/// This does not touch `review_rounds` itself, which stays the operator's
98/// call.
99pub(crate) const STAGNANT_LIMIT: usize = 2;
100
101/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
102/// a base that moved before giving up and leaving the run `Blocked` for a
103/// person.
104///
105/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
106/// that keeps moving faster than a run can catch it is not something more
107/// rebasing fixes, it is a person's call. Not the same *number as*
108/// `land_rounds` - this budget is spent before a pull request exists, land's
109/// after - but bounded for the identical reason, so it uses the same
110/// default. Counted across both call sites in [`Runner::finish_after_tally`]
111/// (once before review, once before the gate), because either one finding
112/// the base still moving is the same signal.
113const BASE_SYNC_ROUNDS: usize = 4;
114
115/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
116/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
117/// reply held no [`FixReport`].
118///
119/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
120/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
121/// "I'll pause here until the `cargo make check` background run reports
122/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
123/// No `FixReport` was ever collected from that seat, and the run moved on to
124/// the next review round regardless.
125///
126/// Bounded independently of `review_rounds` and `graph.retries`: this
127/// recovers one seat's missing report mid-round, not a new round of review or
128/// an ordinary parse retry, and must not itself become the unbounded wait the
129/// rest of this module exists to avoid.
130const MAX_FIX_CONTINUATIONS: usize = 2;
131
132/// One queued agent invocation.
133///
134/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
135/// CLI hung up on its own stream is asked again from the same job rather than
136/// rebuilt from scratch. See [`Runner::resume_undelivered`].
137#[derive(Clone)]
138struct SeatJob {
139    spec: AgentSpec,
140    seat: SeatState,
141    cwd: PathBuf,
142    prompt: String,
143    timeout: Duration,
144    allow_write: bool,
145    sessions: bool,
146    artifacts: PathBuf,
147    stem: String,
148}
149
150/// How the graph reads one agent invocation.
151///
152/// Quota is split out from an ordinary failure on purpose: a rate-limited call
153/// is known to fail again if retried now, so the retry loop must not spend an
154/// attempt on it. `Dropped` is split out for the opposite reason: unlike
155/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
156/// error JSON, never the agent's answer — a caller that matched only
157/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
158/// left to read that JSON as if it were usable output. `resume_undelivered`
159/// is the only caller that acts on it; everywhere else it is reported like an
160/// ordinary failure.
161enum AgentOutcome {
162    /// A usable output.
163    Ok(AgentOutput),
164    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
165    Quota(AgentOutput),
166    /// The CLI hung up on its own stream after billed work. See
167    /// [`agent::AgentOutput::work_undelivered`].
168    Dropped(AgentOutput),
169    /// Any other failure: a timeout, a bad exit code, an empty reply.
170    Failed(String),
171}
172
173/// A request to park the run at its next node boundary.
174///
175/// Cloning is how the request travels: the loop keeps one handle and hands a
176/// clone to each [`Runner`], and every clone points at the same flag. There
177/// is no channel because there is nothing to send - the only message is
178/// "park", it is idempotent, and a flag cannot be missed by a receiver that
179/// was not listening yet.
180///
181/// The boundary is what makes this cheap. Every node writes the run's state
182/// before the next one starts, and every node skips what is already recorded:
183/// `prep` returns early once candidates exist, `implement` asks only the seats
184/// with nothing on disk, `judge` returns early once judgements exist. So a
185/// parked run resumes into exactly the node it stopped before, and no agent
186/// work is thrown away. Killing the process mid-node, by contrast, loses
187/// whatever the seats in flight had not yet written - which for an implement
188/// wave is an hour of paid work.
189///
190/// A [`Runner`] watches two independent handles of this type - see
191/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
192/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
193/// clone covering the whole daemon's lifetime and is never asked to un-park,
194/// which is correct exactly because nothing is dispatched after it fires.
195/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
196/// that parks for an interrupted task must go on to run other tasks
197/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
198/// reusing the daemon-wide one.
199#[derive(Debug, Clone, Default)]
200pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
201
202impl Pause {
203    /// A pause nobody has asked for yet.
204    #[must_use]
205    pub fn new() -> Self {
206        Self::default()
207    }
208
209    /// Ask the run to park at its next node boundary. Idempotent.
210    pub fn park(&self) {
211        self.0.store(true, Ordering::SeqCst);
212    }
213
214    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
215    /// fold into the run's own `park` event - so an operator reading the run
216    /// later knows this was a deliberate interrupt rather than a shutdown or
217    /// a binary swap. The first reason recorded wins; a park already in
218    /// flight is not relabelled by a second, unrelated request.
219    pub fn park_because(&self, reason: impl Into<String>) {
220        let mut reason_guard = self
221            .1
222            .lock()
223            .unwrap_or_else(std::sync::PoisonError::into_inner);
224        if reason_guard.is_none() {
225            *reason_guard = Some(reason.into());
226        }
227        drop(reason_guard);
228        self.park();
229    }
230
231    /// Has a park been asked for?
232    #[must_use]
233    pub fn parked(&self) -> bool {
234        self.0.load(Ordering::SeqCst)
235    }
236
237    /// Why the park was asked for, when the caller used [`Pause::park_because`].
238    #[must_use]
239    pub fn reason(&self) -> Option<String> {
240        self.1
241            .lock()
242            .unwrap_or_else(std::sync::PoisonError::into_inner)
243            .clone()
244    }
245}
246
247/// Drives one run.
248pub struct Runner {
249    /// Run state; public so the CLI can report on it.
250    pub state: RunState,
251    roles: ResolvedRoles,
252    sem: Arc<Semaphore>,
253    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
254    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
255    /// this is never the same handle as `interrupt`.
256    pause: Pause,
257    /// Set when `magi serve`'s interrupt scheduler wants this specific run
258    /// parked at its next node boundary, to let a task marked
259    /// [`crate::queue::Task::interrupt`] run alone before this one carries
260    /// on. Unlike `pause`, a fresh, unshared handle per run - see
261    /// [`Runner::watch_interrupt`].
262    interrupt: Pause,
263}
264
265/// The commit a run branches from: the base branch as the remote has it.
266///
267/// Two failures this replaces. A run used to branch off `HEAD` and so refused
268/// to start on a dirty tree, which made `magi serve` decline every task for as
269/// long as the operator had work in progress - most of the time. Branching off
270/// the *local* base branch fixed that and introduced a worse one: `land` merges
271/// the winner on GitHub, nothing updates the local ref, and the next run
272/// branches off a base missing everything the previous runs landed. Two tasks
273/// in a row from a phone would have had the second silently re-implementing
274/// against stale code and opening a pull request that reverted the first.
275///
276/// Only refs move here - no checkout, no local branch, no merge - so it is safe
277/// with uncommitted work in the tree. A machine with no network still starts:
278/// the fetch may fail and the local tip is used with a warning, because
279/// refusing to run offline is a worse failure than running against a base the
280/// operator can see for themselves.
281///
282/// One function, called by both entry points. Two answers to "where does a run
283/// branch from" is the kind of drift nobody notices until a diff is wrong.
284/// Bring the local `branch` in line with `<remote>/<branch>` before a review
285/// checks it out.
286///
287/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
288/// anything other than plain `git push` from this checkout (a jj colocated
289/// workspace, another clone) moves only the remote-tracking ref - so the local
290/// one can be a stale placeholder. It moves only when local is behind the remote or is an
291/// empty placeholder that diverged from it; unpushed local work is kept, and a real
292/// divergence is refused rather than guessed at.
293async fn sync_review_branch(repo: &Path, branch: &str, remote: &str) -> Result<()> {
294    let tracking = format!("{remote}/{branch}");
295    let fetched = git::fetch(repo, remote, branch).await;
296    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
297    let local_exists = git::branch_exists(repo, branch).await?;
298    if !fresh {
299        if !local_exists {
300            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
301        }
302        tracing::warn!(
303            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
304        );
305        return Ok(());
306    }
307    let remote_sha = git::rev_parse(repo, &tracking).await?;
308    if !local_exists {
309        git::git(repo, &["branch", branch, &tracking]).await?;
310        return Ok(());
311    }
312    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
313    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
314        return Ok(());
315    }
316    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
317        // Diverged. A local tip that adds nothing over the fork point is a
318        // placeholder the remote's work replaced (a jj rewrite of the same
319        // change); anything else is local work we must not discard.
320        let mb = git::git_raw(repo, &["merge-base", &local_sha, &remote_sha]).await?;
321        let placeholder = mb.ok()
322            && git::git_raw(repo, &["diff", "--quiet", &mb.stdout, &local_sha])
323                .await?
324                .ok();
325        if !placeholder {
326            bail!(
327                "local `{branch}` ({}) and {tracking} ({}) have diverged, so it is unclear \
328                 which one to review; reconcile them, e.g. `git branch -f {branch} {tracking}` \
329                 to review the pushed work, or push the local branch first",
330                short(&local_sha),
331                short(&remote_sha)
332            );
333        }
334    }
335    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
336    if !out.ok() {
337        bail!(
338            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
339            short(&local_sha),
340            short(&remote_sha),
341            out.stderr
342        );
343    }
344    tracing::warn!(
345        "local `{branch}` was stale: fast-forwarded {} -> {}",
346        short(&local_sha),
347        short(&remote_sha)
348    );
349    Ok(())
350}
351
352async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
353    let tracking = format!("{remote}/{base_branch}");
354    let fetched = git::fetch(repo, remote, base_branch).await;
355    if let Ok(out) = &fetched
356        && out.ok()
357        && git::rev_exists(repo, &tracking).await
358    {
359        return git::rev_parse(repo, &tracking).await;
360    }
361    let why = match &fetched {
362        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
363        Ok(_) => format!("{remote} has no {base_branch}"),
364        Err(e) => e.to_string(),
365    };
366    tracing::warn!(
367        "could not read {tracking} ({why}); branching off the local \
368         {base_branch} instead, which may be behind"
369    );
370    git::rev_parse(repo, base_branch).await.with_context(|| {
371        format!(
372            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
373             branch that exists"
374        )
375    })
376}
377
378/// Exclusive claim on one run's `magi fix` step, released on drop — including
379/// on an early return or a panic.
380///
381/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
382/// manual `magi fix` invocations against the same run are otherwise
383/// invisible to each other and would race to remove and recreate the same
384/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
385/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
386/// which is only ever reclaimed later, out of band, by
387/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
388/// `magi fix` invocation is not necessarily running under either of those, so
389/// nothing would ever sweep a lock a killed or crashed process left behind.
390/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
391/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
392/// lease use: an unreadable or unparsable pid, or a liveness query the
393/// platform cannot answer, reads as alive and the lock is left in place.
394struct FixClaim {
395    path: PathBuf,
396}
397
398impl FixClaim {
399    fn acquire(dir: &Path) -> Result<Self> {
400        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
401        let path = dir.join("fix.lock");
402        match Self::create(&path) {
403            Ok(claim) => Ok(claim),
404            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
405                if Self::reclaim_if_dead(&path) {
406                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
407                } else {
408                    bail!(
409                        "another `magi fix` is already running for this run ({} exists)",
410                        path.display()
411                    )
412                }
413            }
414            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
415        }
416    }
417
418    fn create(path: &Path) -> std::io::Result<Self> {
419        let mut f = std::fs::OpenOptions::new()
420            .write(true)
421            .create_new(true)
422            .open(path)?;
423        use std::io::Write as _;
424        // Read back by `reclaim_if_dead` on a later, stuck invocation.
425        writeln!(f, "{}", std::process::id())?;
426        Ok(Self {
427            path: path.to_owned(),
428        })
429    }
430
431    /// True if the lock named a process confirmed dead, in which case it was
432    /// also removed. Never true on an unreadable file, an unparsable pid, or
433    /// a liveness query the platform cannot answer — see this type's own doc.
434    fn reclaim_if_dead(path: &Path) -> bool {
435        let dead = std::fs::read_to_string(path)
436            .ok()
437            .and_then(|body| body.trim().parse::<u32>().ok())
438            .is_some_and(|pid| !crate::proc::pid_alive(pid));
439        dead && std::fs::remove_file(path).is_ok()
440    }
441}
442
443impl Drop for FixClaim {
444    fn drop(&mut self) {
445        let _ = std::fs::remove_file(&self.path);
446    }
447}
448
449impl Runner {
450    /// Start a fresh run against `repo`.
451    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
452        let repo = git::toplevel(repo).await?;
453        let missing = agent::missing_programs(&config.agents);
454        if !missing.is_empty() {
455            bail!(
456                "these agent programs are not on PATH: {}. Fix the roster in \
457                 magi.toml or install them.",
458                missing.join(", ")
459            );
460        }
461        let base_branch = match config.merge.base.clone() {
462            Some(b) => b,
463            None => git::current_branch(&repo)
464                .await?
465                .context("HEAD is detached; set [merge] base in magi.toml")?,
466        };
467        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
468        // Still worth saying out loud. The operator's uncommitted work is not
469        // part of this run, and someone watching a candidate fail to use a
470        // change they just made deserves to know why.
471        if !git::is_clean(&repo).await? {
472            tracing::warn!(
473                "{} has uncommitted changes; they are not part of this run, \
474                 which branches off {base_branch} ({})",
475                repo.display(),
476                &base_commit[..base_commit.len().min(8)]
477            );
478        }
479        let roles = config.resolve_roles()?;
480        let max_parallel = config.graph.max_parallel.max(1);
481        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
482        state.event("start", format!("run {} created", state.id));
483        state.save()?;
484        Ok(Self {
485            state,
486            roles,
487            sem: Arc::new(Semaphore::new(max_parallel)),
488            pause: Pause::new(),
489            interrupt: Pause::new(),
490        })
491    }
492
493    /// Open a review-only run against work that already exists on `branch`.
494    ///
495    /// The expensive half of the graph is the implement wave — measured at
496    /// 111 and 134 internal tool-loop turns on this repository, against a
497    /// handful for a judge or a reviewer. The cheap half is worth running on
498    /// hand-written work too, and there was no way to reach it.
499    ///
500    /// No new state and no schema change are needed: a run with **one** viable
501    /// candidate and a tally already decided degrades `execute` to exactly
502    /// review → gate → merge, because `judge` skips a single-candidate field,
503    /// `deliberate` has fewer than two first choices to reconcile, `vote`
504    /// returns early, `tally` is already present and `fold_losers` has no
505    /// losers. Resuming such a run therefore does the right thing as well.
506    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
507        let repo = git::toplevel(repo).await?;
508        let missing = agent::missing_programs(&config.agents);
509        if !missing.is_empty() {
510            bail!(
511                "these agent programs are not on PATH: {}. Fix the roster in \
512                 magi.toml or install them.",
513                missing.join(", ")
514            );
515        }
516        sync_review_branch(&repo, branch, &config.merge.remote).await?;
517        let base_branch = match config.merge.base.clone() {
518            Some(b) => b,
519            None => git::current_branch(&repo)
520                .await?
521                .context("HEAD is detached; set [merge] base in magi.toml")?,
522        };
523        if base_branch == branch {
524            bail!("`{branch}` is the base branch; there is nothing to review against");
525        }
526        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
527
528        let roles = config.resolve_roles()?;
529        let max_parallel = config.graph.max_parallel.max(1);
530        // The commit subjects are the closest thing to a task statement that
531        // existing work carries, and the reviewers are told as much.
532        let log = git::log_oneline(&repo, &base_commit, branch)
533            .await
534            .unwrap_or_default();
535        let instruction = format!(
536            "Review the work already on branch `{branch}`. There is no task \
537             statement: what the change claims to do is whatever its commits \
538             say.\n\n{}",
539            if log.trim().is_empty() {
540                "(no commit messages)"
541            } else {
542                log.trim()
543            }
544        );
545        let mut state = RunState::new(
546            repo.clone(),
547            base_branch,
548            base_commit.clone(),
549            instruction,
550            config,
551        );
552
553        // An attached worktree, so the fixer's commits land on the branch under
554        // review rather than on a detached head nobody will look at again.
555        let worktree = state.worktree_root().join("under-review");
556        if let Some(parent) = worktree.parent() {
557            tokio::fs::create_dir_all(parent).await.ok();
558        }
559        let path = worktree.to_string_lossy().to_string();
560        git::git(&repo, &["worktree", "add", &path, branch])
561            .await
562            .with_context(|| {
563                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
564            })?;
565
566        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
567            .await
568            .unwrap_or(0);
569        if commits == 0 {
570            git::worktree_remove(&repo, &worktree).await.ok();
571            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
572        }
573        let files = git::changed_files(&worktree, &base_commit, "HEAD")
574            .await
575            .map(|f| f.len())
576            .unwrap_or(0);
577        if files == 0
578            && let (Ok(head_tree), Ok(base_tree)) = (
579                git::tree_of(&worktree, "HEAD").await,
580                git::tree_of(&worktree, &base_commit).await,
581            )
582            && head_tree == base_tree
583        {
584            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
585            git::worktree_remove(&repo, &worktree).await.ok();
586            bail!(
587                "`{branch}` at {} has a tree identical to base {}; this usually means \
588                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
589                 against `{}/{branch}`) rather than an empty change",
590                short(&head),
591                short(&base_commit),
592                state.config.merge.remote
593            );
594        }
595        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
596            .await
597            .unwrap_or_default();
598
599        state.candidates.push(Candidate {
600            index: 0,
601            label: 'A',
602            // Not an agent id on purpose: nothing in the roster wrote this, and
603            // the stats tables must not credit anyone with a win for it.
604            agent: "(existing branch)".to_owned(),
605            branch: branch.to_owned(),
606            worktree,
607            summary: String::new(),
608            stat,
609            files,
610            commits,
611            empty: false,
612            failed: None,
613            verified_noop: None,
614            duration_ms: 0,
615            folded: false,
616        });
617        state.tally = Some(Tally {
618            first_choice: BTreeMap::from([('A', 0)]),
619            borda: BTreeMap::new(),
620            winner: 'A',
621            rankings: 0,
622            unanimous_initial: false,
623            deliberated: false,
624            changed_votes: 0,
625            unanimous_final: false,
626            tie_break: None,
627            // No panel sat, so no quorum applies. Zero judges is the correct
628            // number for work that never competed, and must not be reported as
629            // a collapsed panel.
630            judges: 0,
631            present: 0,
632            quorum: 0,
633            met_quorum: true,
634            uncontested: Some("review-only run: nothing competed".to_owned()),
635        });
636        state.status = RunStatus::Reviewing;
637        state.event(
638            "start",
639            format!(
640                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
641                state.id
642            ),
643        );
644        state.save()?;
645        Ok(Self {
646            state,
647            roles,
648            sem: Arc::new(Semaphore::new(max_parallel)),
649            pause: Pause::new(),
650            interrupt: Pause::new(),
651        })
652    }
653
654    /// Reopen an existing run.
655    pub fn resume(id: &str) -> Result<Self> {
656        let state = RunState::load(id)?;
657        let roles = state.config.resolve_roles()?;
658        let max_parallel = state.config.graph.max_parallel.max(1);
659        Ok(Self {
660            state,
661            roles,
662            sem: Arc::new(Semaphore::new(max_parallel)),
663            pause: Pause::new(),
664            interrupt: Pause::new(),
665        })
666    }
667
668    /// Walk the graph to a terminal state, skipping nodes already recorded.
669    ///
670    /// Every way a run is driven - the queue loop, `magi run`, a resume from
671    /// the phone - ends here, so this is the one place a run that ended
672    /// Blocked / Stalled / Failed, or died with an error, is announced to the
673    /// notification centre. Best-effort: see [`crate::notices::raise`].
674    pub async fn execute(&mut self) -> Result<()> {
675        let result = self.execute_graph().await;
676        let ended = if result.is_err() {
677            Some(crate::notices::run_stopped(&self.state.id, &self.state))
678        } else {
679            crate::notices::run_ended(&self.state)
680        };
681        if let Some(notice) = ended {
682            crate::notices::raise(notice);
683        }
684        result
685    }
686
687    async fn execute_graph(&mut self) -> Result<()> {
688        // Moving again, so it is no longer parked. Set before the walk rather
689        // than in `resume`, so every way of re-entering the graph clears it
690        // and a card cannot claim a run is waiting to be resumed while the
691        // agents are already working.
692        self.state.parked = false;
693        // Any seat this state still lists as answering belongs to whatever
694        // process last drove this run — this one included, if it crashed
695        // mid-wave. Cleared and flushed immediately, before anything else
696        // runs, so a resume can never show a seat as live when nothing is
697        // asking it anything yet; the node that actually dispatches the next
698        // wave repopulates it.
699        self.state.clear_active();
700        // Recorded in the same spot, and flushed together with the clear
701        // above: this is the pid a reader checks (`RunState::liveness`) when
702        // no daemon claim exists to answer "is a process still driving this
703        // run" — a plain `magi run` / `magi review` typed into a terminal
704        // claims nothing there. Always overwritten, never only-if-absent, so
705        // a resumed run's stale pid from a previous, possibly-dead process
706        // can never survive into this one's own report. Unlike
707        // `clear_active`, this changes on every single `execute()` call, so
708        // the save below is now unconditional rather than only-if-cleared.
709        //
710        // `driver_started_at` is recorded in the same breath, from this same
711        // pid, so `liveness` can tell a live pid that is genuinely still us
712        // apart from one the OS has since handed to an unrelated process —
713        // see that field's own doc for why the pid alone is not enough.
714        let pid = std::process::id();
715        self.state.driver_pid = Some(pid);
716        self.state.driver_started_at = crate::proc::process_started_at(pid);
717        self.state.save()?;
718        // A run that already lost its quorum never resumes into the verdict
719        // machinery: `deliberate` and `vote` would otherwise clobber the
720        // stalled marker back to Voting and the run would keep going past a
721        // verdict that is no longer trustworthy. Everything already recorded is
722        // kept, so the run stays resumable (or foldable) for a human to pick up.
723        //
724        // On --resume the run gets one chance to repair itself: the seats a
725        // rate limit took out are re-asked. If their quota has since reset and
726        // the quorum is restored, the run picks up and finishes; otherwise it
727        // stays stale and still-resumable for a later retry. If it does not
728        // recover, the returned status stays `Stalled` and nothing was
729        // clobbered (the recovery only mutates entries for the lost seats).
730        if self.state.status == RunStatus::Stalled {
731            if self.recover_stall().await? {
732                self.finish_after_tally().await?;
733            } else {
734                // Still below quorum: persist the marker and stay resumable.
735                self.state.save()?;
736            }
737            return Ok(());
738        }
739        // A run parked inside `land` - watching CI, mid fix-round, or
740        // waiting on the owner's merge approval - resumes directly into it,
741        // never back through `prep`. Everything before `merge` already
742        // concluded; that is the only way `status` reaches `Landing` in the
743        // first place. Re-walking `review_loop` first would also be actively
744        // wrong: its own status recomputation (see its doc) treats any
745        // clean round as reason to set `status` to `Gating`, which would
746        // clobber this marker before `merge` ever ran, and this run would
747        // never find its way back into `land` at all.
748        if self.state.status == RunStatus::Landing {
749            self.run_land().await?;
750            // `run_land` may have settled the run right here - CI came back
751            // green and the PR merged, say - without ever passing back
752            // through `merge`'s own trailing call. Whatever it left `status`
753            // as is what this has to read.
754            self.settle_questions();
755            return Ok(());
756        }
757        self.prep().await?;
758        if self.park_here()? {
759            return Ok(());
760        }
761        self.advise().await?;
762        if self.park_here()? {
763            return Ok(());
764        }
765        self.implement().await?;
766        if self.park_here()? {
767            return Ok(());
768        }
769        // `after_implement` already saved the state and settled any open
770        // questions when it set this; nothing later in the graph has
771        // anything to judge.
772        if self.state.status == RunStatus::VerifiedNoop {
773            return Ok(());
774        }
775        self.judge().await?;
776        if self.park_here()? {
777            return Ok(());
778        }
779        self.deliberate().await?;
780        if self.park_here()? {
781            return Ok(());
782        }
783        self.vote().await?;
784        if self.park_here()? {
785            return Ok(());
786        }
787        self.tally()?;
788        // A verdict that lost its quorum is not trustworthy: do not review,
789        // gate, or merge on it. Everything already done is kept, so the run
790        // stays resumable (or foldable); the human can replace the agent that
791        // ran out of quota and pick it up.
792        if self.state.status == RunStatus::Stalled {
793            // Persist the stalled marker now — the normal end-of-execute save
794            // below is below this early return, and without it a resumed run
795            // would reload a pre-tally status and keep going.
796            self.state.save()?;
797            return Ok(());
798        }
799        self.finish_after_tally().await?;
800        Ok(())
801    }
802
803    /// Park here if asked to, recording it in the run's own timeline.
804    ///
805    /// Returns whether the caller should stop walking the graph. The state is
806    /// saved either way by the node that just finished; this adds the event so
807    /// the operator's card says why a run that is neither finished nor moving
808    /// is sitting where it is.
809    fn park_here(&mut self) -> Result<bool> {
810        // Either handle asking is enough - see `Pause`'s own doc for why
811        // they are never the same one. `interrupt` is checked second so a
812        // reason it carries is preferred in the message below over a plain
813        // shutdown park racing it at the same boundary.
814        if !self.pause.parked() && !self.interrupt.parked() {
815            return Ok(false);
816        }
817        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
818            Some(reason) => format!(
819                "parked after `{}` ({reason}) — resume to carry on from here",
820                self.state.status.as_str()
821            ),
822            None => format!(
823                "parked after `{}` — resume to carry on from here",
824                self.state.status.as_str()
825            ),
826        };
827        self.state.event("park", why);
828        self.state.parked = true;
829        self.state.save()?;
830        Ok(true)
831    }
832
833    /// Hand the runner the pause `magi serve`'s own shutdown watches.
834    pub fn on_pause(&mut self, pause: Pause) {
835        self.pause = pause;
836    }
837
838    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
839    /// scheduler asking this one run - and no other - to park so a task
840    /// marked [`crate::queue::Task::interrupt`] can run alone. See
841    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
842    /// handle.
843    pub fn watch_interrupt(&mut self, pause: Pause) {
844        self.interrupt = pause;
845    }
846
847    /// Abandon this run's own open questions, once `status` has actually
848    /// settled rather than merely paused.
849    ///
850    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
851    /// either back up with the candidates, the review round and the seat
852    /// sessions already on disk, so a question an implementer asked mid-round
853    /// may still get a real answer read by a real resume. Only the statuses
854    /// `resumable` excludes are actually final: the run merged, it reached
855    /// `Ready` with nothing left to do, it failed outright with no
856    /// established point to continue from, or every candidate agreed, with
857    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
858    /// every one of those the seat that asked is gone for good, exactly like
859    /// the run being deleted under `magi run rm` - so the same cleanup
860    /// applies, worded for what actually happened instead of "the run was
861    /// deleted".
862    ///
863    /// Best-effort and silent on success: called from every place `status`
864    /// can land on one of those three, including ones a resumed run revisits,
865    /// so it must cost nothing when there was nothing open to begin with.
866    fn settle_questions(&mut self) {
867        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
868            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
869        }
870    }
871
872    /// The tail of the graph after a trustworthy tally: fold losers, review,
873    /// gate, merge, and persist.
874    async fn finish_after_tally(&mut self) -> Result<()> {
875        self.fold_losers().await?;
876        // Before review starts, and again right before the gate: a run's
877        // review rounds can themselves take long enough for the base to move
878        // a second time, and the gate is the one node whose "green" gets
879        // acted on.
880        self.sync_to_base().await?;
881        self.review_loop().await?;
882        self.sync_to_base().await?;
883        self.gate().await?;
884        self.merge().await?;
885        self.state.save()?;
886        Ok(())
887    }
888
889    // ---------------------------------------------------------------- prep
890
891    async fn prep(&mut self) -> Result<()> {
892        if !self.state.candidates.is_empty() {
893            return Ok(());
894        }
895        self.state.status = RunStatus::Prep;
896        let repo = self.state.repo.clone();
897        let base = self.state.base_commit.clone();
898        let root = self.state.worktree_root();
899        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
900
901        // The hook is the write-time half of the blindness contract; the
902        // presentation filter in `blind` is the half that cannot be bypassed.
903        let hooks_dir = self.state.dir().join("hooks");
904        if self.state.config.blind.commit_msg_hook {
905            std::fs::create_dir_all(&hooks_dir)
906                .with_context(|| format!("create {}", hooks_dir.display()))?;
907            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
908            let path = hooks_dir.join("commit-msg");
909            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
910            make_executable(&path)?;
911            // Ref-counted rather than a plain idempotent set: with more than
912            // one run able to be in flight in the same repository at once
913            // (see `Config::daemon.max_concurrent_runs`), a bare "already
914            // true?" check cannot tell "another run of mine still needs
915            // this" from "nobody does", and the run that happens to finish
916            // first would disable the hook out from under a sibling still
917            // relying on it.
918            git::acquire_worktree_config(&repo).await?;
919            self.state.enabled_worktree_config = true;
920        }
921
922        for (index, (spec, label)) in self
923            .roles
924            .implementers
925            .clone()
926            .into_iter()
927            .zip(labels)
928            .enumerate()
929        {
930            let branch = self.state.branch_for(label);
931            let worktree = root.join(format!("cand-{label}"));
932            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
933            if self.state.config.blind.commit_msg_hook {
934                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
935            }
936            git::local_exclude(&worktree, "/.magi/").await?;
937            self.state.candidates.push(Candidate {
938                index,
939                label,
940                agent: spec.id.clone(),
941                branch,
942                worktree,
943                summary: String::new(),
944                stat: String::new(),
945                files: 0,
946                commits: 0,
947                empty: false,
948                failed: None,
949                verified_noop: None,
950                duration_ms: 0,
951                folded: false,
952            });
953        }
954
955        for j in 1..=self.roles.judges.len() {
956            let wt = root.join(format!("judge-{j}"));
957            if !wt.exists() {
958                git::worktree_add_detached(&repo, &wt, &base).await?;
959            }
960        }
961
962        // Disposable, detached checkouts for the design-deliberation stage's
963        // advisor seats — the same shape as the judges' above, at the same
964        // base commit, since advisors also only ever read. Sized off the
965        // configured count directly rather than a resolved roster: unlike
966        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
967        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
968        // `prep` has no `ResolvedRoles` field to read a count from here.
969        if self.state.config.graph.advise {
970            for k in 1..=self.state.config.graph.advisors {
971                let wt = root.join(format!("advisor-{k}"));
972                if !wt.exists() {
973                    git::worktree_add_detached(&repo, &wt, &base).await?;
974                }
975            }
976        }
977
978        // A judge cannot tell it is looking at its own patch — the seats keep
979        // separate conversations — but a panel that shares agents with the
980        // field is less independent than it looks, and that is worth saying out
981        // loud once per run rather than leaving it in the config.
982        let authors: Vec<&str> = self
983            .roles
984            .implementers
985            .iter()
986            .map(|a| a.id.as_str())
987            .collect();
988        let overlap: Vec<String> = self
989            .roles
990            .judges
991            .iter()
992            .enumerate()
993            .filter(|(_, j)| authors.contains(&j.id.as_str()))
994            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
995            .collect();
996        if !overlap.is_empty() {
997            let note = format!(
998                "{} also authored a candidate; blind, but the panel is less \
999                 independent than {} distinct agents would be",
1000                overlap.join(", "),
1001                self.roles.judges.len()
1002            );
1003            self.state.event("prep", note);
1004        }
1005
1006        self.state.event(
1007            "prep",
1008            format!(
1009                "{} candidates, {} judges, base {} ({})",
1010                self.state.candidates.len(),
1011                self.roles.judges.len(),
1012                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1013                self.state.base_branch
1014            ),
1015        );
1016        self.state.status = RunStatus::Implementing;
1017        self.state.save()?;
1018        Ok(())
1019    }
1020
1021    // -------------------------------------------------------------- advise
1022
1023    /// The design-deliberation stage: independent, read-only advisor seats
1024    /// each sketch a design before any implementer touches the repository,
1025    /// and (when at least one produced a usable proposal) a synthesis seat
1026    /// blends them into a brief `implement` carries in every candidate's
1027    /// prompt.
1028    ///
1029    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1030    /// advisors` is the proposal count. Everything here is best-effort and
1031    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1032    /// that cannot reach quota, or a synthesis seat that produced nothing
1033    /// usable all leave `implement` exactly as it was before this stage
1034    /// existed — the task instruction alone — rather than failing the whole
1035    /// competition over an enrichment stage. Every outcome is still recorded
1036    /// as an event, so a run that got nothing from this stage says why.
1037    ///
1038    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1039    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1040    /// resumed run whose stage failed would re-run it, and re-spend the
1041    /// agent calls, on every reentry before `implement`.
1042    ///
1043    /// Also skipped once any candidate shows implementation progress — the
1044    /// exact predicate `implement` itself uses to decide a candidate is no
1045    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1046    /// is not enough: a run created by an older binary that predates this
1047    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1048    /// an already-`Implementing`-or-later run under this build would
1049    /// otherwise walk straight back through `prep` (a no-op once candidates
1050    /// exist) into this node and spawn every advisor seat against worktrees
1051    /// `prep` never recreated — after implementation has already started,
1052    /// which is exactly the invariant this stage exists to guarantee.
1053    async fn advise(&mut self) -> Result<()> {
1054        let implement_untouched = self
1055            .state
1056            .candidates
1057            .iter()
1058            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1059        if !self.state.config.graph.advise || self.state.advise_attempted {
1060            return Ok(());
1061        }
1062        if !implement_untouched {
1063            self.state.event(
1064                "advise",
1065                "skipping the design-deliberation stage: at least one \
1066                 candidate already shows implementation progress, so this \
1067                 run is past the point the stage exists to run before"
1068                    .to_owned(),
1069            );
1070            self.state.advise_attempted = true;
1071            self.state.save()?;
1072            return Ok(());
1073        }
1074        let run_id = self.state.id.clone();
1075        let prompts = self.state.config.prompts.clone();
1076        let instruction = self.state.instruction.clone();
1077        let language = self.state.config.graph.language.clone();
1078        let root = self.state.worktree_root();
1079        let n = self.state.config.graph.advisors;
1080        let where_recorded = self.state.dir().join("run.json");
1081
1082        let seats = match self.state.config.advisors() {
1083            Ok(seats) if !seats.is_empty() => seats,
1084            Ok(_) => {
1085                self.state.event(
1086                    "advise",
1087                    format!(
1088                        "[graph] advisors is 0; skipping the design-deliberation \
1089                         stage and continuing without a synthesis brief (see {})",
1090                        where_recorded.display()
1091                    ),
1092                );
1093                self.state.advise_attempted = true;
1094                self.state.save()?;
1095                return Ok(());
1096            }
1097            Err(e) => {
1098                self.state.event(
1099                    "advise",
1100                    format!(
1101                        "could not resolve advisor seats ({e:#}); continuing \
1102                         without a design-deliberation brief (see {})",
1103                        where_recorded.display()
1104                    ),
1105                );
1106                self.state.advise_attempted = true;
1107                self.state.save()?;
1108                return Ok(());
1109            }
1110        };
1111
1112        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1113        let artifacts = agent::artifacts_dir(&self.state.dir());
1114        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1115
1116        let mut jobs = Vec::new();
1117        for (i, spec) in seats.iter().cloned().enumerate() {
1118            let seat_key = format!("advisor-{}", i + 1);
1119            let seat = self.seat(&seat_key, &spec.id);
1120            jobs.push(SeatJob {
1121                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1122                spec,
1123                seat,
1124                cwd: worktrees[i % worktrees.len()].clone(),
1125                timeout,
1126                allow_write: false,
1127                sessions: false,
1128                artifacts: artifacts.clone(),
1129                stem: seat_key,
1130            });
1131        }
1132
1133        self.state.event(
1134            "advise",
1135            format!(
1136                "{} advisor seat(s) sketching a design in parallel",
1137                jobs.len()
1138            ),
1139        );
1140        let mut quota_losses = Vec::new();
1141        let cache = self.state.config.cache_dir();
1142        let ctx = WaveCtx {
1143            run: &run_id,
1144            node: "advise",
1145            prompts: &prompts,
1146            cache: cache.as_deref(),
1147            round: None,
1148        };
1149        let results = ask_json_wave::<Proposal>(
1150            jobs,
1151            Arc::clone(&self.sem),
1152            self.state.config.graph.retries,
1153            &ctx,
1154            &mut quota_losses,
1155            &mut self.state,
1156            &|p: &Proposal| p.validate(),
1157        )
1158        .await;
1159        self.state.quota.extend(quota_losses);
1160
1161        let mut records = Vec::with_capacity(results.len());
1162        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1163            let agent_id = seat.agent.clone();
1164            self.state.seats.insert(seat.key.clone(), seat);
1165            match res {
1166                Ok((proposal, out)) => {
1167                    self.state
1168                        .event("advise", format!("advisor-{} proposed a design", i + 1));
1169                    records.push(advise::AdvisorRecord::proposed(
1170                        i + 1,
1171                        agent_id,
1172                        proposal,
1173                        out.duration_ms,
1174                    ));
1175                }
1176                Err(e) => {
1177                    self.state.event(
1178                        "advise",
1179                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1180                    );
1181                    records.push(advise::AdvisorRecord::failed(
1182                        i + 1,
1183                        agent_id,
1184                        e.to_string(),
1185                    ));
1186                }
1187            }
1188        }
1189
1190        let mut advice = advise::Advice {
1191            records,
1192            synthesis: None,
1193        };
1194        if advice.proposals().is_empty() {
1195            self.state.event(
1196                "advise",
1197                "no advisor produced a usable proposal; continuing without a \
1198                 synthesis brief"
1199                    .to_owned(),
1200            );
1201        } else {
1202            match self
1203                .synthesize_brief(
1204                    &advice,
1205                    &instruction,
1206                    &language,
1207                    &worktrees[0],
1208                    &artifacts,
1209                    &run_id,
1210                    &prompts,
1211                    cache.as_deref(),
1212                )
1213                .await
1214            {
1215                Ok(Some(text)) => {
1216                    self.state.event(
1217                        "advise",
1218                        "synthesized a design brief for the implementer".to_owned(),
1219                    );
1220                    advice.synthesis = Some(text);
1221                }
1222                Ok(None) => {
1223                    self.state.event(
1224                        "advise",
1225                        "the synthesis seat produced nothing usable; continuing \
1226                         without a design brief"
1227                            .to_owned(),
1228                    );
1229                }
1230                Err(e) => {
1231                    self.state.event(
1232                        "advise",
1233                        format!("could not synthesize a design brief: {e:#}"),
1234                    );
1235                }
1236            }
1237        }
1238        advise::apply_reflection(&mut advice);
1239
1240        self.state.advice = Some(advice);
1241        self.state.advise_attempted = true;
1242        self.state.save()?;
1243        Ok(())
1244    }
1245
1246    /// The synthesis seat: reads every advisor's proposal and blends them
1247    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1248    /// out of [`Runner::advise`] only for readability — it is not called
1249    /// anywhere else.
1250    ///
1251    /// Picked the same way [`crate::talk`]'s standing conversation and
1252    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1253    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1254    /// order (a claude seat, else the first runnable agent in roster order)
1255    /// used when that field is unset — see `[roles] synthesizer`'s own doc
1256    /// in [`crate::config`] for why a dedicated field exists here at all.
1257    #[allow(clippy::too_many_arguments)]
1258    async fn synthesize_brief(
1259        &mut self,
1260        advice: &advise::Advice,
1261        instruction: &str,
1262        language: &str,
1263        cwd: &Path,
1264        artifacts: &Path,
1265        run_id: &str,
1266        prompts: &Prompts,
1267        cache: Option<&Path>,
1268    ) -> Result<Option<String>> {
1269        let want = self.state.config.roles.synthesizer.as_deref();
1270        let spec = agent::pick(&self.state.config.agents, want, &agent::installed)?;
1271        let mut seat = self.seat("advise-synthesis", &spec.id);
1272        let proposals = advice.proposals();
1273        let mut prompt = prompt::with_overlay(
1274            prompt::synthesize_brief(instruction, &proposals, language),
1275            prompts.overlay("advise"),
1276        );
1277        if cache.is_some() {
1278            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1279            // below — see `prompt::build_cache_note`'s doc for why telling a
1280            // read-only seat to build through the shared cache is exactly how
1281            // a sandbox's write refusal gets misread as a defect.
1282            prompt.push('\n');
1283            prompt.push_str(&prompt::build_cache_note("advise", false));
1284        }
1285        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1286        let out = agent::invoke(
1287            &spec,
1288            &mut seat,
1289            &Invocation {
1290                cwd,
1291                prompt: &prompt,
1292                timeout,
1293                allow_write: false,
1294                sessions: false,
1295                artifacts,
1296                stem: "advise-synthesis",
1297                run: run_id,
1298                node: "advise",
1299                cache_dir: None,
1300                attachments: &[],
1301            },
1302        )
1303        .await?;
1304        self.state.seats.insert(seat.key.clone(), seat);
1305        if !out.usable() {
1306            return Ok(None);
1307        }
1308        let text =
1309            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1310        Ok((!text.trim().is_empty()).then_some(text))
1311    }
1312
1313    // ----------------------------------------------------------- implement
1314
1315    async fn implement(&mut self) -> Result<()> {
1316        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1317        // agent files with `magi task add` name the run that paid for it. The
1318        // prompt overlay is cloned alongside it because the waves borrow it
1319        // while `self` is mutably borrowed by the node's own bookkeeping.
1320        let run_id = self.state.id.clone();
1321        let prompts = self.state.config.prompts.clone();
1322        let todo: Vec<usize> = self
1323            .state
1324            .candidates
1325            .iter()
1326            .enumerate()
1327            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1328            .map(|(i, _)| i)
1329            .collect();
1330        if todo.is_empty() {
1331            return self.after_implement();
1332        }
1333        self.state.status = RunStatus::Implementing;
1334
1335        let language = self.state.config.graph.language.clone();
1336        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1337        let sessions = self.state.config.graph.sessions;
1338        let artifacts = agent::artifacts_dir(&self.state.dir());
1339        // The design-deliberation stage's blended brief, when `advise` found
1340        // one — carried into every implementer's prompt the same way
1341        // regardless of which candidate it is.
1342        let brief = self
1343            .state
1344            .advice
1345            .as_ref()
1346            .and_then(|a| a.synthesis.as_deref())
1347            .map(str::to_owned);
1348
1349        let mut jobs = Vec::new();
1350        for &i in &todo {
1351            let (index, label, worktree) = {
1352                let c = &self.state.candidates[i];
1353                (c.index, c.label, c.worktree.clone())
1354            };
1355            let spec = self.roles.implementers[index].clone();
1356            let seat_key = format!("impl-{label}");
1357            let seat = self.seat(&seat_key, &spec.id);
1358            let instruction = self.state.instruction.clone();
1359            jobs.push(SeatJob {
1360                spec,
1361                seat,
1362                prompt: prompt::implement(
1363                    &instruction,
1364                    &worktree.to_string_lossy(),
1365                    &language,
1366                    brief.as_deref(),
1367                ),
1368                cwd: worktree,
1369                timeout,
1370                allow_write: true,
1371                sessions,
1372                artifacts: artifacts.clone(),
1373                stem: format!("impl-{label}"),
1374            });
1375        }
1376
1377        self.state.event(
1378            "implement",
1379            format!("{} candidates in parallel", jobs.len()),
1380        );
1381        // Kept so a seat whose CLI hung up can be asked again from the same
1382        // job: `wave` consumes what it is given. Mutable so `resume_quota_losses`
1383        // can update a seat's own entry once a fallback agent takes it over —
1384        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1385        // whichever agent actually answered, not the one that quota'd out.
1386        let mut sent = jobs.clone();
1387        let cache = self.state.config.cache_dir();
1388        let ctx = WaveCtx {
1389            run: &run_id,
1390            node: "implement",
1391            prompts: &prompts,
1392            cache: cache.as_deref(),
1393            round: None,
1394        };
1395        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1396        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1397            .await;
1398        self.resume_quota_losses(&mut results, &mut sent, &prompts, &run_id)
1399            .await;
1400        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1401            .await;
1402
1403        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1404            let seat_key = seat.key.clone();
1405            // A quota fallback (`resume_quota_losses`) may have handed this
1406            // seat to a different agent than the one `prep` recorded on the
1407            // candidate; the stats tables and any later fixer-defaults-to-
1408            // winner's-author lookup must credit whoever actually answered —
1409            // unless every fallback also quota'd out, in which case nobody
1410            // actually answered and crediting the last agent tried would
1411            // erase every earlier agent's own quota loss from the stats
1412            // tables instead of just this one seat's.
1413            let agent = seat.agent.clone();
1414            let exhausted_the_fallback_chain = matches!(&out, AgentOutcome::Quota(_));
1415            self.state.seats.insert(seat.key.clone(), seat);
1416            let label = self.state.candidates[i].label;
1417            let worktree = self.state.candidates[i].worktree.clone();
1418            let base = self.state.base_commit.clone();
1419
1420            let (summary, duration, failed, verified_claim) = match out {
1421                AgentOutcome::Ok(o) => {
1422                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1423                    let failed = (!o.usable()).then(|| {
1424                        if o.timed_out {
1425                            "agent timed out".to_owned()
1426                        } else {
1427                            format!("agent exited with {:?}", o.exit_code)
1428                        }
1429                    });
1430                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1431                    (text, o.duration_ms, failed, verified_claim)
1432                }
1433                // Left un-resumed by `resume_undelivered` (a dirty tree
1434                // already rescues the work, or there was no session left to
1435                // resume into) — reported like the ordinary failure it is,
1436                // never as if `o.text` (the CLI's raw error JSON) were an
1437                // answer.
1438                AgentOutcome::Dropped(o) => {
1439                    let why = o
1440                        .dropped
1441                        .as_ref()
1442                        .map(|d| d.why.as_str())
1443                        .unwrap_or("the CLI ended the stream without delivering its answer");
1444                    (
1445                        String::new(),
1446                        o.duration_ms,
1447                        Some(format!("the CLI dropped the stream ({why})")),
1448                        None,
1449                    )
1450                }
1451                AgentOutcome::Quota(o) => {
1452                    self.state.quota.push(QuotaLoss {
1453                        seat: seat_key,
1454                        node: "implement".to_owned(),
1455                        at: Timestamp::now(),
1456                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1457                    });
1458                    (
1459                        String::new(),
1460                        o.duration_ms,
1461                        Some("rate limited (quota); produced no change".to_owned()),
1462                        None,
1463                    )
1464                }
1465                AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1466            };
1467
1468            // Rescue anything the agent edited but never committed: an
1469            // uncommitted candidate would silently be an empty one.
1470            let rescued = match git::rescue_commit(
1471                &worktree,
1472                &format!("magi: candidate {label} (uncommitted work)"),
1473            )
1474            .await
1475            {
1476                Ok(r) => {
1477                    self.state.note_withheld("implement", &r.withheld);
1478                    r.committed
1479                }
1480                Err(_) => false,
1481            };
1482            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1483                .await
1484                .unwrap_or(0);
1485            let patch = git::diff(&worktree, &base, "HEAD")
1486                .await
1487                .unwrap_or_default();
1488            let stat = git::diff_stat(&worktree, &base, "HEAD")
1489                .await
1490                .unwrap_or_default();
1491            let files = git::changed_files(&worktree, &base, "HEAD")
1492                .await
1493                .map(|f| f.len())
1494                .unwrap_or(0);
1495            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1496
1497            let c = &mut self.state.candidates[i];
1498            if !exhausted_the_fallback_chain {
1499                c.agent = agent;
1500            }
1501            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1502            c.stat = stat;
1503            c.files = files;
1504            c.commits = commits;
1505            c.duration_ms = duration;
1506            c.empty = commits == 0 || patch.trim().is_empty();
1507            // An agent that failed but still produced a committed change stays
1508            // in the running: the patch is what gets judged, not the exit code.
1509            c.failed = match failed {
1510                Some(_) if c.empty => failed,
1511                _ => None,
1512            };
1513            // Only an empty candidate can be a verified no-op: a claim next
1514            // to a real patch is not what the marker is for, and `c.failed`
1515            // being `Some` here already implies `verified_claim` was never
1516            // set (see the guard above the match that produced it).
1517            c.verified_noop = if c.empty { verified_claim } else { None };
1518            let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1519                (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1520                (None, true, Some(_), _) => {
1521                    format!("candidate {label}: no change produced (agent-verified no-op)")
1522                }
1523                (None, true, None, _) => format!("candidate {label}: no change produced"),
1524                (None, false, _, true) => {
1525                    format!(
1526                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1527                    )
1528                }
1529                (None, false, _, false) => {
1530                    format!("candidate {label}: {files} files, {commits} commits")
1531                }
1532            };
1533            self.state.event("implement", note);
1534            self.state.save()?;
1535        }
1536
1537        self.after_implement()
1538    }
1539
1540    /// Ask again, once, for work a CLI did and then failed to hand over.
1541    ///
1542    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1543    /// status with an empty response and a usage report showing output tokens,
1544    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1545    /// seven minutes and 14,267 output tokens that arrived as an empty
1546    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1547    ///
1548    /// Two conditions, and both matter:
1549    ///
1550    /// - **Only when the tree is untouched.** Often the agent has already
1551    ///   written its files and only the closing message was lost; the rescue
1552    ///   commit below picks that up and there is nothing to ask for. Re-asking
1553    ///   then would pay for a second implementation of work already on disk.
1554    /// - **Once.** A CLI that drops one stream can drop the next, and this
1555    ///   node is the most expensive in the graph.
1556    ///
1557    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1558    /// dropped reply still carried its `conversation_id`, so the seat is asked
1559    /// to finish what it was doing rather than sent the whole task again. It
1560    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1561    /// node's - for the same reason a re-ranked judge does: restating finished
1562    /// work is not the work.
1563    ///
1564    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1565    /// same way until it resets, while an abandoned conversation is still
1566    /// there to be picked up.
1567    async fn resume_undelivered(
1568        &mut self,
1569        results: &mut [(usize, SeatState, AgentOutcome)],
1570        sent: &[SeatJob],
1571        prompts: &Prompts,
1572        run_id: &str,
1573    ) {
1574        for (wi, seat, out) in results.iter_mut() {
1575            let Some(dropped) = (match &*out {
1576                AgentOutcome::Dropped(o) => o.dropped.clone(),
1577                _ => None,
1578            }) else {
1579                continue;
1580            };
1581            let Some(job) = sent.get(*wi) else { continue };
1582            // Already on disk? Then only the closing message was lost.
1583            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1584                self.state.event(
1585                    "implement",
1586                    format!(
1587                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1588                         work is in the tree",
1589                        seat.key, dropped.output_tokens, dropped.why
1590                    ),
1591                );
1592                continue;
1593            }
1594            // The re-ask only makes sense as a resume: `resume_after_drop`
1595            // says nothing about the task, trusting the seat to still hold it.
1596            // Without a session to resume — sessions disabled, or this CLI's
1597            // drop shape happened not to carry a session id — that prompt
1598            // would open a brand-new conversation with no context at all,
1599            // which is worse than leaving this as the ordinary failure it
1600            // already is.
1601            if !has_context(&job.spec, seat, job.sessions) {
1602                self.state.event(
1603                    "implement",
1604                    format!(
1605                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1606                         is no session left to resume",
1607                        seat.key, dropped.output_tokens, dropped.why
1608                    ),
1609                );
1610                continue;
1611            }
1612            self.state.event(
1613                "implement",
1614                format!(
1615                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1616                     conversation",
1617                    seat.key, dropped.output_tokens, dropped.why
1618                ),
1619            );
1620            let mut retry = job.clone();
1621            retry.seat = seat.clone();
1622            retry.prompt = prompt::resume_after_drop(&dropped.why);
1623            retry.timeout = retry_budget(job.timeout, true);
1624            retry.stem = format!("{}-resume", job.stem);
1625            let cache = self.state.config.cache_dir();
1626            let ctx = WaveCtx {
1627                run: run_id,
1628                node: "implement",
1629                prompts,
1630                cache: cache.as_deref(),
1631                round: None,
1632            };
1633            let (resumed_seat, resumed) =
1634                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1635            *seat = resumed_seat;
1636            *out = resumed;
1637        }
1638    }
1639
1640    /// Fall an implement seat through to the next untried agent in the
1641    /// implementer roster when it lost to quota, instead of leaving the
1642    /// seat's loss final the moment one agent's account runs dry.
1643    ///
1644    /// Solo runs (`graph.candidates = 1`, `daemon::apply_solo`'s forced shape)
1645    /// are the motivating case: `Config::resolve_roles`'s `implementers`
1646    /// truncates to the single slot rotation picked, so a solo task whose one
1647    /// implementer hits quota mid-run used to have nothing else to try. This
1648    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
1649    /// unrotated roster — which is the only place the *other* candidates in
1650    /// the machine's roster still exist once `implementers` has been cut down
1651    /// to size.
1652    ///
1653    /// Walks forward from just past the seat's own original position in the
1654    /// roster, never wrapping back to the front: a later candidate slot (say
1655    /// `beta`, the roster's second entry) must fall through to the *next*
1656    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
1657    /// almost certainly a different candidate's own agent already — and once
1658    /// the roster's tail is exhausted there is nothing left to fall through
1659    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
1660    /// whole [`AgentSpec`]: a roster with the same id named twice must not
1661    /// let this retry that id forever. The loop keeps falling through until
1662    /// an attempt lands something other than `Quota` or the roster's tail
1663    /// runs out of untried ids, at which point the seat is left exactly as
1664    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
1665    /// `QuotaLoss` recorded, the candidate failed/empty.
1666    ///
1667    /// `sent` is taken mutably and updated with the fallback agent's spec:
1668    /// `resume_unconfirmed_commands`, which runs after this and also reads
1669    /// `sent`, must see whichever agent actually ended up answering the seat
1670    /// — reading the stale, original spec there would check session
1671    /// eligibility against the wrong CLI and could hand a fallback agent's
1672    /// session id to the agent that just lost the seat to quota.
1673    ///
1674    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
1675    /// — `self.seat` only reuses state when the agent id is unchanged, so
1676    /// handing it a different id already gets this for free. Reusing the old
1677    /// seat would resume a different CLI's session as if it were a
1678    /// continuation of this one.
1679    ///
1680    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
1681    /// a quota loss cuts an agent off mid-turn, so anything already in the
1682    /// tree is unfinished work, not a completed candidate a re-ask would pay
1683    /// for twice. A dirty tree is rescued into a commit first (the same
1684    /// neutral-identity rescue `implement`'s own outcome loop gives every
1685    /// candidate) so the next agent starts clean.
1686    ///
1687    /// The new agent gets the implementer's full prompt and full
1688    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
1689    /// it has no session and no context, and is implementing the task from
1690    /// nothing, unlike a resumed drop which is only restating work already
1691    /// done.
1692    ///
1693    /// Every intermediate `Quota` this loop absorbs is folded into a plain
1694    /// `implement` event, never into `self.state.quota` — that is what
1695    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
1696    /// go unspent, and a seat that ultimately recovered on its second or
1697    /// third agent is not the stalled panel that check exists to catch. Only
1698    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
1699    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
1700    /// outcome loop already has — this helper never pushes to it itself.
1701    async fn resume_quota_losses(
1702        &mut self,
1703        results: &mut [(usize, SeatState, AgentOutcome)],
1704        sent: &mut [SeatJob],
1705        prompts: &Prompts,
1706        run_id: &str,
1707    ) {
1708        let instruction = self.state.instruction.clone();
1709        let language = self.state.config.graph.language.clone();
1710        let brief = self
1711            .state
1712            .advice
1713            .as_ref()
1714            .and_then(|a| a.synthesis.as_deref())
1715            .map(str::to_owned);
1716        for (wi, seat, out) in results.iter_mut() {
1717            let Some(job) = sent.get_mut(*wi) else {
1718                continue;
1719            };
1720            // Where the seat's own original agent sits in the roster — the
1721            // fallback walk starts just past here, never at the front, so a
1722            // later candidate slot's quota loss does not fall back onto an
1723            // earlier slot's own agent.
1724            let start = self
1725                .roles
1726                .implementer_roster
1727                .iter()
1728                .position(|s| s.id == job.spec.id)
1729                .unwrap_or(0);
1730            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1731            let mut fallback_attempt = 0usize;
1732            while matches!(&*out, AgentOutcome::Quota(_)) {
1733                let Some(next) =
1734                    next_untried_implementer(&self.roles.implementer_roster, start, &tried)
1735                        .cloned()
1736                else {
1737                    break;
1738                };
1739                tried.insert(next.id.clone());
1740                fallback_attempt += 1;
1741
1742                if let Ok(r) = git::rescue_commit(
1743                    &job.cwd,
1744                    &format!(
1745                        "magi: candidate {} (uncommitted work before quota fallback)",
1746                        seat.key
1747                    ),
1748                )
1749                .await
1750                {
1751                    self.state.note_withheld("implement", &r.withheld);
1752                }
1753
1754                self.state.event(
1755                    "implement",
1756                    format!(
1757                        "{}: rate limited (quota) on {}; retrying with {}",
1758                        seat.key, seat.agent, next.id
1759                    ),
1760                );
1761
1762                let new_seat = self.seat(&seat.key, &next.id);
1763                // Kept in sync on `sent` itself, not just the local retry: a
1764                // later helper (`resume_unconfirmed_commands`) reads `sent`
1765                // after this one returns and must see whichever agent is now
1766                // occupying the seat, not the one that just quota'd out —
1767                // otherwise it would judge session/continuation eligibility
1768                // by the wrong CLI and could resend a fallback's session id
1769                // to the agent that lost it the seat in the first place.
1770                job.spec = next.clone();
1771                let mut retry = job.clone();
1772                retry.seat = new_seat;
1773                retry.prompt = prompt::implement(
1774                    &instruction,
1775                    &job.cwd.to_string_lossy(),
1776                    &language,
1777                    brief.as_deref(),
1778                );
1779                retry.stem = format!("{}-quota-{}", job.stem, next.id);
1780                let cache = self.state.config.cache_dir();
1781                let ctx = WaveCtx {
1782                    run: run_id,
1783                    node: "implement",
1784                    prompts,
1785                    cache: cache.as_deref(),
1786                    round: None,
1787                };
1788                let (fallback_seat, fallback_out) = run_one(
1789                    retry,
1790                    Arc::clone(&self.sem),
1791                    &ctx,
1792                    &mut self.state,
1793                    fallback_attempt,
1794                )
1795                .await;
1796                *seat = fallback_seat;
1797                *out = fallback_out;
1798            }
1799        }
1800    }
1801
1802    /// Ask an implement seat's own CLI to confirm what it started, once, when
1803    /// its reply reported a command whose completion status it never
1804    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1805    /// that does and does not mean.
1806    ///
1807    /// The completion contract this task asks for, extended to `implement`
1808    /// with the same signal `continue_fix_report` reads for the fixer,
1809    /// rather than a keyword search over the reply or a hard requirement on
1810    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1811    /// a candidate's CLI turn ended cleanly while a test run it had started
1812    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1813    /// named in it at all is untouched by this: `commands` is empty, so
1814    /// there is nothing to be unconfirmed.
1815    ///
1816    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1817    /// this is not about recovering edits that might already be on disk, it
1818    /// is about a result the seat itself never vouched for, which resuming
1819    /// asks for regardless of what the tree already holds. Bounded to one
1820    /// attempt for the same reason `resume_undelivered` is — this is the
1821    /// most expensive node in the graph — and a seat that still cannot
1822    /// confirm on that attempt is left as whatever its (possibly still
1823    /// unconfirmed) reply says; this does not invent a new "failed" reason
1824    /// for a candidate that otherwise produced a real, committed change.
1825    async fn resume_unconfirmed_commands(
1826        &mut self,
1827        results: &mut [(usize, SeatState, AgentOutcome)],
1828        sent: &[SeatJob],
1829        prompts: &Prompts,
1830        run_id: &str,
1831    ) {
1832        for (wi, seat, out) in results.iter_mut() {
1833            let AgentOutcome::Ok(o) = &*out else {
1834                continue;
1835            };
1836            if !has_unconfirmed_command(&o.commands) {
1837                continue;
1838            }
1839            let Some(job) = sent.get(*wi) else { continue };
1840            if !has_context(&job.spec, seat, job.sessions) {
1841                self.state.event(
1842                    "implement",
1843                    format!(
1844                        "{}: the reply named a command whose own CLI never confirmed the exit \
1845                         status of, but there is no session left to resume",
1846                        seat.key
1847                    ),
1848                );
1849                continue;
1850            }
1851            self.state.event(
1852                "implement",
1853                format!(
1854                    "{}: the reply named a command whose own CLI never confirmed the exit \
1855                     status of; resuming the conversation",
1856                    seat.key
1857                ),
1858            );
1859            let mut retry = job.clone();
1860            retry.seat = seat.clone();
1861            retry.prompt = prompt::resume_incomplete(
1862                "a command in your last reply had no confirmed exit status",
1863            );
1864            retry.timeout = retry_budget(job.timeout, true);
1865            retry.stem = format!("{}-confirm", job.stem);
1866            let cache = self.state.config.cache_dir();
1867            let ctx = WaveCtx {
1868                run: run_id,
1869                node: "implement",
1870                prompts,
1871                cache: cache.as_deref(),
1872                round: None,
1873            };
1874            let (resumed_seat, resumed) =
1875                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1876            *seat = resumed_seat;
1877            *out = resumed;
1878        }
1879    }
1880
1881    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1882    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1883    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1884    /// that motivated this.
1885    ///
1886    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1887    /// own nudge loop already covers for judge/review/vote seats, and not a
1888    /// dropped stream, which [`Runner::resume_undelivered`] covers for
1889    /// implement seats: here the CLI turn genuinely finished while the node's
1890    /// own work — the fixer's account of what it did — had not. Gated purely
1891    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
1892    /// reply, never on any wording in it, so a fixer whose valid, first-try
1893    /// `FixReport` happens to mention having waited on a background test is
1894    /// never resumed — the `Ok(report)` branch at the call site returns
1895    /// before this is ever invoked.
1896    ///
1897    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
1898    /// attempt ([`retry_budget`]), nothing attempted once the session is
1899    /// gone, and a quota hit ends the loop immediately rather than retrying a
1900    /// rate limit that fails the same way again.
1901    async fn continue_fix_report(
1902        &mut self,
1903        mut seat: SeatState,
1904        parse_err: String,
1905        job: &SeatJob,
1906        prompts: &Prompts,
1907        run_id: &str,
1908        round: usize,
1909    ) -> (
1910        SeatState,
1911        Option<FixReport>,
1912        Option<String>,
1913        ContinuationRecord,
1914    ) {
1915        let mut last_err = parse_err;
1916        let mut cumulative_wait_ms = 0u64;
1917        let mut attempts = 0usize;
1918        loop {
1919            if !has_context(&job.spec, &seat, job.sessions) {
1920                self.state.event(
1921                    "fix",
1922                    format!(
1923                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
1924                         session left to resume into"
1925                    ),
1926                );
1927                let outcome = if attempts == 0 {
1928                    ContinuationOutcome::NoSession
1929                } else {
1930                    ContinuationOutcome::Exhausted
1931                };
1932                return (
1933                    seat,
1934                    None,
1935                    Some(format!("unparsable fix report: {last_err}")),
1936                    ContinuationRecord {
1937                        attempts,
1938                        cumulative_wait_ms,
1939                        outcome,
1940                    },
1941                );
1942            }
1943            if attempts >= MAX_FIX_CONTINUATIONS {
1944                self.state.event(
1945                    "fix",
1946                    format!(
1947                        "round {round}: fixer's reply still had no adoption report after \
1948                         {attempts} continuation(s) ({last_err}); giving up"
1949                    ),
1950                );
1951                return (
1952                    seat,
1953                    None,
1954                    Some(format!(
1955                        "unparsable fix report after {attempts} continuation(s): {last_err}"
1956                    )),
1957                    ContinuationRecord {
1958                        attempts,
1959                        cumulative_wait_ms,
1960                        outcome: ContinuationOutcome::Exhausted,
1961                    },
1962                );
1963            }
1964            attempts += 1;
1965            self.state.event(
1966                "fix",
1967                format!(
1968                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
1969                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
1970                ),
1971            );
1972            let mut retry = job.clone();
1973            retry.seat = seat.clone();
1974            retry.prompt = prompt::resume_incomplete(&last_err);
1975            retry.timeout = retry_budget(job.timeout, true);
1976            retry.stem = format!("{}-continue{attempts}", job.stem);
1977            let cache = self.state.config.cache_dir();
1978            let ctx = WaveCtx {
1979                run: run_id,
1980                node: "fix",
1981                prompts,
1982                cache: cache.as_deref(),
1983                round: Some(round),
1984            };
1985            let (resumed_seat, resumed_out) = run_one(
1986                retry,
1987                Arc::clone(&self.sem),
1988                &ctx,
1989                &mut self.state,
1990                attempts,
1991            )
1992            .await;
1993            seat = resumed_seat;
1994            match resumed_out {
1995                AgentOutcome::Ok(o) => {
1996                    cumulative_wait_ms += o.duration_ms;
1997                    match verdict::extract_json::<FixReport>(&o.text) {
1998                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
1999                            self.state.event(
2000                                "fix",
2001                                format!(
2002                                    "round {round}: fixer's adoption report recovered after \
2003                                     {attempts} continuation(s)"
2004                                ),
2005                            );
2006                            return (
2007                                seat,
2008                                Some(report),
2009                                None,
2010                                ContinuationRecord {
2011                                    attempts,
2012                                    cumulative_wait_ms,
2013                                    outcome: ContinuationOutcome::Resumed,
2014                                },
2015                            );
2016                        }
2017                        // The report parsed, but this same reply's own
2018                        // CommandEvidence — the identical record `state.jobs`
2019                        // renders — names a command whose CLI never
2020                        // confirmed an exit status. Read together, that is
2021                        // not a resolved answer: keep nudging rather than
2022                        // accept a report standing next to a command the
2023                        // seat's own CLI cannot vouch for.
2024                        Ok(_) => {
2025                            last_err = "the reply parsed, but it reported a command whose own CLI \
2026                                 never confirmed an exit status"
2027                                .to_owned();
2028                        }
2029                        Err(e) => last_err = e.to_string(),
2030                    }
2031                }
2032                AgentOutcome::Quota(o) => {
2033                    cumulative_wait_ms += o.duration_ms;
2034                    self.state.quota.push(QuotaLoss {
2035                        seat: seat.key.clone(),
2036                        node: "fix".to_owned(),
2037                        at: Timestamp::now(),
2038                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2039                    });
2040                    self.state.event(
2041                        "fix",
2042                        format!(
2043                            "round {round}: continuation rate limited (quota); not retrying now"
2044                        ),
2045                    );
2046                    return (
2047                        seat,
2048                        None,
2049                        Some("rate limited (quota) while recovering the fix report".to_owned()),
2050                        ContinuationRecord {
2051                            attempts,
2052                            cumulative_wait_ms,
2053                            outcome: ContinuationOutcome::QuotaLost,
2054                        },
2055                    );
2056                }
2057                AgentOutcome::Dropped(o) => {
2058                    cumulative_wait_ms += o.duration_ms;
2059                    let why = o
2060                        .dropped
2061                        .as_ref()
2062                        .map(|d| d.why.as_str())
2063                        .unwrap_or("the CLI ended the stream without delivering its answer");
2064                    last_err = format!("the CLI dropped the stream ({why})");
2065                }
2066                AgentOutcome::Failed(e) => last_err = e,
2067            }
2068        }
2069    }
2070
2071    fn after_implement(&mut self) -> Result<()> {
2072        // Scan every candidate patch once the set is complete.
2073        if self.state.leaks.is_empty() {
2074            let cfg = self.state.config.blind.clone();
2075            let mut leaks = Vec::new();
2076            for c in &self.state.candidates {
2077                let Some(patch) =
2078                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2079                else {
2080                    continue;
2081                };
2082                leaks.extend(blind::scan(
2083                    &format!("candidate {} patch", c.label),
2084                    &patch,
2085                    &cfg.vendor_tokens,
2086                ));
2087            }
2088            if !leaks.is_empty() {
2089                let summary = leaks
2090                    .iter()
2091                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2092                    .collect::<Vec<_>>()
2093                    .join(", ");
2094                match cfg.on_leak {
2095                    LeakPolicy::Fail => {
2096                        self.state.status = RunStatus::Failed;
2097                        self.state
2098                            .event("blind", format!("vendor text in a patch: {summary}"));
2099                        self.state.leaks = leaks;
2100                        self.state.save()?;
2101                        self.settle_questions();
2102                        bail!(
2103                            "blind.on_leak = \"fail\" and vendor text reached a \
2104                             judged patch: {summary}"
2105                        );
2106                    }
2107                    LeakPolicy::Redact => self.state.event(
2108                        "blind",
2109                        format!("redacting vendor text for judging: {summary}"),
2110                    ),
2111                    LeakPolicy::Warn => self.state.event(
2112                        "blind",
2113                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
2114                    ),
2115                }
2116                self.state.leaks = leaks;
2117            }
2118        }
2119
2120        if self.state.viable().is_empty() {
2121            if self.state.all_candidates_verified_noop() {
2122                // Every candidate agreed, with evidence the adoption guard
2123                // accepted, that nothing belongs in this worktree. That is
2124                // not the same fact as a candidate that simply failed to
2125                // write anything, and settling it as an ordinary `Failed`
2126                // (see `SCHEMA`'s doc for schema 10) is what let two of
2127                // task 391f's attempts burn a retry each re-discovering the
2128                // same already-landed fix. Terminal either way, so `judge`
2129                // must never run over an empty candidate set — unlike the
2130                // `Failed` branch below this returns `Ok`, not an error:
2131                // nothing here failed.
2132                self.state.status = RunStatus::VerifiedNoop;
2133                self.state.save()?;
2134                self.settle_questions();
2135                return Ok(());
2136            }
2137            self.state.status = RunStatus::Failed;
2138            self.state.save()?;
2139            self.settle_questions();
2140            bail!("no candidate produced a change; nothing to judge");
2141        }
2142        self.state.status = RunStatus::Judging;
2143        self.state.save()?;
2144        Ok(())
2145    }
2146
2147    // --------------------------------------------------------------- judge
2148
2149    async fn judge(&mut self) -> Result<()> {
2150        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2151        // agent files with `magi task add` name the run that paid for it. The
2152        // prompt overlay is cloned alongside it because the waves borrow it
2153        // while `self` is mutably borrowed by the node's own bookkeeping.
2154        let run_id = self.state.id.clone();
2155        let prompts = self.state.config.prompts.clone();
2156        if !self.state.judgements.is_empty() || self.state.judge_skipped {
2157            return Ok(());
2158        }
2159        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2160        if viable.len() == 1 {
2161            // Recorded so this is a one-time event: `judgements` stays empty
2162            // either way, which without this flag is indistinguishable from
2163            // "not yet judged" on the next reentry — and status is left
2164            // untouched, so a later node's conclusion (e.g. `Blocked` after
2165            // the review budget ran out) survives a resume instead of being
2166            // clobbered back to `Judging` by this node running again.
2167            self.state.judge_skipped = true;
2168            self.state.event(
2169                "judge",
2170                format!(
2171                    "only candidate {} produced a change; judging skipped",
2172                    viable[0].label
2173                ),
2174            );
2175            self.state.save()?;
2176            return Ok(());
2177        }
2178        self.state.status = RunStatus::Judging;
2179
2180        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2181        let language = self.state.config.graph.language.clone();
2182        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2183        let sessions = self.state.config.graph.sessions;
2184        let artifacts = agent::artifacts_dir(&self.state.dir());
2185        let root = self.state.worktree_root();
2186        let base_short = short(&self.state.base_commit);
2187
2188        let mut jobs = Vec::new();
2189        let mut orders = Vec::new();
2190        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2191            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2192            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2193            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2194            let seat_key = format!("judge-{}", j + 1);
2195            let seat = self.seat(&seat_key, &spec.id);
2196            jobs.push(SeatJob {
2197                prompt: prompt::judge(
2198                    &self.state.instruction,
2199                    &views,
2200                    self.roles.judges.len(),
2201                    &base_short,
2202                    &language,
2203                ),
2204                spec,
2205                seat,
2206                cwd: root.join(format!("judge-{}", j + 1)),
2207                timeout,
2208                allow_write: false,
2209                sessions,
2210                artifacts: artifacts.clone(),
2211                stem: format!("judge-{}", j + 1),
2212            });
2213        }
2214
2215        self.state.event(
2216            "judge",
2217            format!(
2218                "{} judges ranking {} candidates blind",
2219                jobs.len(),
2220                viable.len()
2221            ),
2222        );
2223        let labels_for_check = labels.clone();
2224        let mut quota_losses = Vec::new();
2225        let cache = self.state.config.cache_dir();
2226        let ctx = WaveCtx {
2227            run: &run_id,
2228            node: "judge",
2229            prompts: &prompts,
2230            cache: cache.as_deref(),
2231            round: None,
2232        };
2233        let results = ask_json_wave::<Ranking>(
2234            jobs,
2235            Arc::clone(&self.sem),
2236            self.state.config.graph.retries,
2237            &ctx,
2238            &mut quota_losses,
2239            &mut self.state,
2240            &move |r: &Ranking| r.validate(&labels_for_check),
2241        )
2242        .await;
2243        self.state.quota.extend(quota_losses);
2244
2245        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2246            let agent_id = seat.agent.clone();
2247            self.state.seats.insert(seat.key.clone(), seat);
2248            let mut record = Judgement {
2249                judge: j + 1,
2250                seat: format!("judge-{}", j + 1),
2251                agent: agent_id,
2252                ranking: Vec::new(),
2253                reasons: BTreeMap::new(),
2254                confidence: None,
2255                order: orders[j].clone(),
2256                failed: None,
2257                duration_ms: 0,
2258            };
2259            match res {
2260                Ok((ranking, out)) => {
2261                    record.ranking = ranking.normalized();
2262                    record.reasons = ranking.reasons;
2263                    record.confidence = ranking.confidence;
2264                    record.duration_ms = out.duration_ms;
2265                    self.state.event(
2266                        "judge",
2267                        format!(
2268                            "judge {} ranked {}",
2269                            j + 1,
2270                            record.ranking.iter().collect::<String>()
2271                        ),
2272                    );
2273                }
2274                Err(e) => {
2275                    record.failed = Some(e.to_string());
2276                    self.state
2277                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2278                }
2279            }
2280            self.state.judgements.push(record);
2281            self.state.save()?;
2282        }
2283        Ok(())
2284    }
2285
2286    // ---------------------------------------------------------- deliberate
2287
2288    async fn deliberate(&mut self) -> Result<()> {
2289        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2290        // agent files with `magi task add` name the run that paid for it. The
2291        // prompt overlay is cloned alongside it because the waves borrow it
2292        // while `self` is mutably borrowed by the node's own bookkeeping.
2293        let run_id = self.state.id.clone();
2294        let prompts = self.state.config.prompts.clone();
2295        if !self.state.deliberation.is_empty() {
2296            return Ok(());
2297        }
2298        let tops: Vec<char> = self
2299            .state
2300            .judgements
2301            .iter()
2302            .filter_map(|j| j.ranking.first().copied())
2303            .collect();
2304        let rounds = self.state.config.graph.deliberate_rounds;
2305        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2306            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2307                self.state.event(
2308                    "deliberate",
2309                    format!("judges agreed on {} outright; no deliberation", tops[0]),
2310                );
2311            }
2312            self.state.status = RunStatus::Voting;
2313            self.state.save()?;
2314            return Ok(());
2315        }
2316
2317        self.state.status = RunStatus::Deliberating;
2318        self.state.event(
2319            "deliberate",
2320            format!(
2321                "split: first choices were {} — opening {rounds} round(s)",
2322                tops.iter().collect::<String>()
2323            ),
2324        );
2325
2326        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2327        let language = self.state.config.graph.language.clone();
2328        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2329        let sessions = self.state.config.graph.sessions;
2330        let artifacts = agent::artifacts_dir(&self.state.dir());
2331        let root = self.state.worktree_root();
2332        let base_short = short(&self.state.base_commit);
2333
2334        // Judges argue in sequence so that a turn can answer the one before it;
2335        // that is the difference between deliberation and three parallel
2336        // monologues.
2337        for round in 1..=rounds {
2338            let mut turns: Vec<DeliberationTurn> = Vec::new();
2339            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2340                if self.state.judgements[j].failed.is_some() {
2341                    continue;
2342                }
2343                let seat_key = format!("judge-{}", j + 1);
2344                let mut seat = self.seat(&seat_key, &spec.id);
2345                let transcript = self.transcript(&turns, j);
2346                let context = if has_context(&spec, &seat, sessions) {
2347                    None
2348                } else {
2349                    Some(self.candidate_block(&viable, &base_short))
2350                };
2351                let text = prompt::deliberate(
2352                    &self.state.instruction,
2353                    context.as_deref(),
2354                    &transcript,
2355                    round,
2356                    rounds,
2357                    &language,
2358                );
2359                let job = SeatJob {
2360                    spec,
2361                    seat: seat.clone(),
2362                    prompt: text,
2363                    cwd: root.join(format!("judge-{}", j + 1)),
2364                    timeout,
2365                    allow_write: false,
2366                    sessions,
2367                    artifacts: artifacts.clone(),
2368                    stem: format!("delib-{round}-judge-{}", j + 1),
2369                };
2370                let cache = self.state.config.cache_dir();
2371                let ctx = WaveCtx {
2372                    run: &run_id,
2373                    node: "deliberate",
2374                    prompts: &prompts,
2375                    cache: cache.as_deref(),
2376                    round: None,
2377                };
2378                let (updated, out) =
2379                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2380                seat = updated;
2381                let agent_id = seat.agent.clone();
2382                let seat_key = seat.key.clone();
2383                self.state.seats.insert(seat.key.clone(), seat);
2384                let body = match out {
2385                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
2386                    // Never read the CLI's raw error JSON as this judge's
2387                    // position — skip the seat instead, the same as any other
2388                    // failed turn.
2389                    AgentOutcome::Dropped(o) => {
2390                        let why =
2391                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
2392                                "the CLI ended the stream without delivering its answer",
2393                            );
2394                        self.state.event(
2395                            "deliberate",
2396                            format!(
2397                                "judge {} skipped: the CLI dropped the stream ({why})",
2398                                j + 1
2399                            ),
2400                        );
2401                        continue;
2402                    }
2403                    AgentOutcome::Quota(o) => {
2404                        self.state.quota.push(QuotaLoss {
2405                            seat: seat_key,
2406                            node: "deliberate".to_owned(),
2407                            at: Timestamp::now(),
2408                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2409                        });
2410                        self.state.event(
2411                            "deliberate",
2412                            format!("judge {} skipped: rate limited (quota)", j + 1),
2413                        );
2414                        continue;
2415                    }
2416                    AgentOutcome::Failed(e) => {
2417                        self.state
2418                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2419                        continue;
2420                    }
2421                };
2422                let tentative = verdict::extract_json::<Position>(&body)
2423                    .ok()
2424                    .and_then(|p| p.tentative)
2425                    .and_then(|s| s.trim().chars().next())
2426                    .map(|c| c.to_ascii_uppercase());
2427                self.state.event(
2428                    "deliberate",
2429                    format!(
2430                        "round {round}: judge {} now favours {}",
2431                        j + 1,
2432                        tentative.map_or("—".to_owned(), |c| c.to_string())
2433                    ),
2434                );
2435                turns.push(DeliberationTurn {
2436                    judge: j + 1,
2437                    agent: agent_id,
2438                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2439                    tentative,
2440                });
2441            }
2442            self.state
2443                .deliberation
2444                .push(DeliberationRound { round, turns });
2445            self.state.save()?;
2446        }
2447
2448        self.state.status = RunStatus::Voting;
2449        self.state.save()?;
2450        Ok(())
2451    }
2452
2453    // ---------------------------------------------------------------- vote
2454
2455    async fn vote(&mut self) -> Result<()> {
2456        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2457        // agent files with `magi task add` name the run that paid for it. The
2458        // prompt overlay is cloned alongside it because the waves borrow it
2459        // while `self` is mutably borrowed by the node's own bookkeeping.
2460        let run_id = self.state.id.clone();
2461        let prompts = self.state.config.prompts.clone();
2462        if !self.state.votes.is_empty() {
2463            return Ok(());
2464        }
2465        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2466        if viable.len() == 1 {
2467            return Ok(());
2468        }
2469        self.state.status = RunStatus::Voting;
2470
2471        let language = self.state.config.graph.language.clone();
2472        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2473        let sessions = self.state.config.graph.sessions;
2474        let artifacts = agent::artifacts_dir(&self.state.dir());
2475        let root = self.state.worktree_root();
2476        let base_short = short(&self.state.base_commit);
2477        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2478
2479        let mut jobs = Vec::new();
2480        let mut seats_at = Vec::new();
2481        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2482            if self
2483                .state
2484                .judgements
2485                .get(j)
2486                .is_some_and(|r| r.failed.is_some())
2487            {
2488                continue;
2489            }
2490            let seat_key = format!("judge-{}", j + 1);
2491            let seat = self.seat(&seat_key, &spec.id);
2492            let mut text = prompt::final_vote(&viable, &language);
2493            if !has_context(&spec, &seat, sessions) {
2494                text = format!(
2495                    "{}\n\n# Candidates\n\n{}",
2496                    text,
2497                    self.candidate_block(&candidates, &base_short)
2498                );
2499            }
2500            jobs.push(SeatJob {
2501                spec,
2502                seat,
2503                prompt: text,
2504                cwd: root.join(format!("judge-{}", j + 1)),
2505                timeout,
2506                allow_write: false,
2507                sessions,
2508                artifacts: artifacts.clone(),
2509                stem: format!("vote-judge-{}", j + 1),
2510            });
2511            seats_at.push(j);
2512        }
2513
2514        self.state.event(
2515            "vote",
2516            format!(
2517                "collecting {} final votes one by one, privately",
2518                jobs.len()
2519            ),
2520        );
2521        let allowed = viable.clone();
2522        let mut quota_losses = Vec::new();
2523        let cache = self.state.config.cache_dir();
2524        let ctx = WaveCtx {
2525            run: &run_id,
2526            node: "vote",
2527            prompts: &prompts,
2528            cache: cache.as_deref(),
2529            round: None,
2530        };
2531        let results = ask_json_wave::<FinalVote>(
2532            jobs,
2533            Arc::clone(&self.sem),
2534            self.state.config.graph.retries,
2535            &ctx,
2536            &mut quota_losses,
2537            &mut self.state,
2538            &move |v: &FinalVote| match v.label() {
2539                Some(c) if allowed.contains(&c) => Ok(()),
2540                other => bail!("vote {other:?} is not one of {allowed:?}"),
2541            },
2542        )
2543        .await;
2544        self.state.quota.extend(quota_losses);
2545
2546        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2547            let agent_id = seat.agent.clone();
2548            self.state.seats.insert(seat.key.clone(), seat);
2549            let initial = self
2550                .state
2551                .judgements
2552                .get(j)
2553                .and_then(|r| r.ranking.first().copied());
2554            let mut record = VoteRecord {
2555                judge: j + 1,
2556                agent: agent_id,
2557                vote: None,
2558                reason: String::new(),
2559                changed: false,
2560            };
2561            match res {
2562                Ok((v, _)) => {
2563                    record.vote = v.label();
2564                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2565                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2566                    self.state.event(
2567                        "vote",
2568                        format!(
2569                            "judge {} voted {}{}",
2570                            j + 1,
2571                            record.vote.unwrap_or('?'),
2572                            if record.changed { " (changed)" } else { "" }
2573                        ),
2574                    );
2575                }
2576                Err(e) => {
2577                    self.state
2578                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2579                }
2580            }
2581            self.state.votes.push(record);
2582            self.state.save()?;
2583        }
2584        Ok(())
2585    }
2586
2587    // --------------------------------------------------------------- tally
2588
2589    fn tally(&mut self) -> Result<()> {
2590        if self.state.tally.is_some() {
2591            return Ok(());
2592        }
2593        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2594        let tops: Vec<char> = self
2595            .state
2596            .judgements
2597            .iter()
2598            .filter_map(|j| j.ranking.first().copied())
2599            .collect();
2600        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2601
2602        // A judge whose private vote failed still counted once, in the initial
2603        // ranking; using it beats discarding a whole seat.
2604        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2605        let mut cast: Vec<char> = Vec::new();
2606        for (i, j) in self.state.judgements.iter().enumerate() {
2607            let vote = self
2608                .state
2609                .votes
2610                .iter()
2611                .find(|v| v.judge == i + 1)
2612                .and_then(|v| v.vote)
2613                .or_else(|| j.ranking.first().copied());
2614            if let Some(v) = vote {
2615                *first_choice.entry(v).or_insert(0) += 1;
2616                cast.push(v);
2617            }
2618        }
2619
2620        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2621        for j in &self.state.judgements {
2622            let n = j.ranking.len();
2623            for (pos, label) in j.ranking.iter().enumerate() {
2624                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2625            }
2626        }
2627
2628        let best = first_choice.values().copied().max().unwrap_or(0);
2629        let mut leaders: Vec<char> = first_choice
2630            .iter()
2631            .filter(|(_, v)| **v == best)
2632            .map(|(k, _)| *k)
2633            .collect();
2634        let mut tie_break = None;
2635        if leaders.len() > 1 {
2636            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2637            let borda_leaders: Vec<char> = leaders
2638                .iter()
2639                .copied()
2640                .filter(|l| borda[l] == top_borda)
2641                .collect();
2642            tie_break = Some(if borda_leaders.len() == 1 {
2643                format!(
2644                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2645                    leaders.len()
2646                )
2647            } else {
2648                format!(
2649                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2650                    leaders.len()
2651                )
2652            });
2653            leaders = borda_leaders;
2654            leaders.sort_unstable();
2655        }
2656        let winner = *leaders
2657            .first()
2658            .or(viable.first())
2659            .context("no candidate to declare a winner from")?;
2660
2661        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2662        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2663        let deliberated = !self.state.deliberation.is_empty();
2664
2665        // Whose verdict is this? A rate-limited seat is absent even if it
2666        // ranked before the limit hit, so presence is measured against the
2667        // recorded losses, not just "did a ranking ever appear".
2668        let quota_seats: std::collections::BTreeSet<&str> =
2669            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2670        let mut present = 0usize;
2671        for (i, j) in self.state.judgements.iter().enumerate() {
2672            if quota_seats.contains(j.seat.as_str()) {
2673                continue;
2674            }
2675            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2676            let voted = self
2677                .state
2678                .votes
2679                .iter()
2680                .any(|v| v.judge == i + 1 && v.vote.is_some());
2681            if ranked || voted {
2682                present += 1;
2683            }
2684        }
2685        // Strict majority of the configured panel. A bare majority is real
2686        // signal we can act on, while a minority verdict must never stand in
2687        // for a healthy one. A one-candidate run needs no panel at all, and
2688        // `judges` stays `0` rather than the roster size a panel that never
2689        // sat would otherwise be credited with.
2690        let needs_quorum = viable.len() > 1;
2691        let judges_total = if needs_quorum {
2692            self.roles.judges.len()
2693        } else {
2694            0
2695        };
2696        let quorum = if needs_quorum {
2697            judges_total / 2 + 1
2698        } else {
2699            0
2700        };
2701        let met_quorum = !needs_quorum || present >= quorum;
2702        let uncontested = (!needs_quorum).then(|| {
2703            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2704        });
2705
2706        self.state.event(
2707            "tally",
2708            match &uncontested {
2709                Some(reason) => format!("winner {winner} — {reason}"),
2710                None => format!(
2711                    "winner {winner} — votes {} | initial {} | {} changed | \
2712                     {present}/{judges_total} judges{}",
2713                    first_choice
2714                        .iter()
2715                        .map(|(k, v)| format!("{k}:{v}"))
2716                        .collect::<Vec<_>>()
2717                        .join(" "),
2718                    if unanimous_initial {
2719                        "unanimous"
2720                    } else {
2721                        "split"
2722                    },
2723                    changed_votes,
2724                    if met_quorum {
2725                        String::new()
2726                    } else {
2727                        format!(" — below quorum ({quorum} required)")
2728                    },
2729                ),
2730            },
2731        );
2732        if !met_quorum {
2733            self.state.event(
2734                "stall",
2735                format!(
2736                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2737                     the run stops here, resumable"
2738                ),
2739            );
2740        }
2741        self.state.tally = Some(Tally {
2742            first_choice,
2743            borda,
2744            winner,
2745            rankings: tops.len(),
2746            unanimous_initial,
2747            deliberated,
2748            changed_votes,
2749            unanimous_final,
2750            tie_break,
2751            judges: judges_total,
2752            present,
2753            quorum,
2754            met_quorum,
2755            uncontested,
2756        });
2757        self.state.status = if met_quorum {
2758            RunStatus::Reviewing
2759        } else {
2760            RunStatus::Stalled
2761        };
2762        self.state.save()?;
2763        Ok(())
2764    }
2765
2766    // ------------------------------------------------------------- recover
2767
2768    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2769    /// resumed toward completion once the transient cause clears.
2770    ///
2771    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2772    /// it toward the quorum, which is exactly the set of seats whose absence
2773    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2774    /// not depend on which node happened to hit the limit), or an ordinary
2775    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2776    /// seat is never disturbed.
2777    ///
2778    /// A seat that now answers with a usable ranking is "recovered": its
2779    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2780    /// `tally` counts it present again), and its vote re-collected. A seat that
2781    /// still fails keeps its loss and stays absent.
2782    ///
2783    /// Returns `true` when the re-tally restores the quorum (the run may proceed
2784    /// to review/gate/merge), `false` when it is still below quorum (the run
2785    /// stays `Stalled`, still resumable for a later retry).
2786    #[allow(clippy::too_many_lines)]
2787    async fn recover_stall(&mut self) -> Result<bool> {
2788        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2789        // agent files with `magi task add` name the run that paid for it. The
2790        // prompt overlay is cloned alongside it because the waves borrow it
2791        // while `self` is mutably borrowed by the node's own bookkeeping.
2792        let run_id = self.state.id.clone();
2793        let prompts = self.state.config.prompts.clone();
2794        // Absent seats = quota-lost at any node, or failed outright. Mirroring
2795        // `tally`'s presence test (rather than the old quota-judge/vote filter)
2796        // is what keeps a non-quota collapse — or a quota loss recorded at the
2797        // deliberate node — from being a permanent dead-end on `--resume`.
2798        let quota_seats: BTreeSet<&str> =
2799            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2800        let absent: Vec<String> = self
2801            .state
2802            .judgements
2803            .iter()
2804            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2805            .map(|j| j.seat.clone())
2806            .collect();
2807        if absent.is_empty() {
2808            return Ok(false);
2809        }
2810        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2811        if viable.len() <= 1 {
2812            return Ok(false);
2813        }
2814        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2815        let language = self.state.config.graph.language.clone();
2816        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2817        let sessions = self.state.config.graph.sessions;
2818        let artifacts = agent::artifacts_dir(&self.state.dir());
2819        let root = self.state.worktree_root();
2820        let base_short = short(&self.state.base_commit);
2821        let candidates: Vec<Candidate> = viable.clone();
2822
2823        // Map each absent seat key to its 0-based position in `roles.judges`.
2824        let mut positions: Vec<usize> = absent
2825            .iter()
2826            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2827            .collect();
2828        if positions.is_empty() {
2829            return Ok(false);
2830        }
2831        positions.sort_unstable();
2832        positions.dedup();
2833
2834        // Re-rank the lost seats, one blind prompt each.
2835        let mut judge_jobs = Vec::new();
2836        for &j in &positions {
2837            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2838            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2839            let seat_key = format!("judge-{}", j + 1);
2840            let spec = self.roles.judges[j].clone();
2841            let seat = self.seat(&seat_key, &spec.id);
2842            judge_jobs.push(SeatJob {
2843                spec,
2844                seat,
2845                prompt: prompt::judge(
2846                    &self.state.instruction,
2847                    &views,
2848                    self.roles.judges.len(),
2849                    &base_short,
2850                    &language,
2851                ),
2852                cwd: root.join(seat_key),
2853                timeout,
2854                allow_write: false,
2855                sessions,
2856                artifacts: artifacts.clone(),
2857                stem: format!("judge-{}-recover", j + 1),
2858            });
2859        }
2860
2861        let labels_for_check = labels.clone();
2862        let mut judge_losses = Vec::new();
2863        let retries = self.state.config.graph.retries;
2864        let cache = self.state.config.cache_dir();
2865        let ctx = WaveCtx {
2866            run: &run_id,
2867            node: "judge",
2868            prompts: &prompts,
2869            cache: cache.as_deref(),
2870            round: None,
2871        };
2872        let results = ask_json_wave::<Ranking>(
2873            judge_jobs,
2874            Arc::clone(&self.sem),
2875            retries,
2876            &ctx,
2877            &mut judge_losses,
2878            &mut self.state,
2879            &move |r: &Ranking| r.validate(&labels_for_check),
2880        )
2881        .await;
2882
2883        // Refresh the judgement of every seat that ranked again.
2884        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2885        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
2886            self.state.seats.insert(seat.key.clone(), seat);
2887            let record = &mut self.state.judgements[j];
2888            match res {
2889                Ok((ranking, out)) => {
2890                    record.ranking = ranking.normalized();
2891                    record.reasons = ranking.reasons;
2892                    record.confidence = ranking.confidence;
2893                    record.failed = None;
2894                    record.duration_ms = out.duration_ms;
2895                    recovered.insert(j);
2896                    self.state.event(
2897                        "recover",
2898                        format!("judge {} ranked again after the limit", j + 1),
2899                    );
2900                }
2901                Err(e) => {
2902                    self.state
2903                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2904                }
2905            }
2906        }
2907
2908        // Re-ask the votes of the seats that recovered a ranking.
2909        let mut vote_jobs = Vec::new();
2910        let mut vote_pos: Vec<usize> = Vec::new();
2911        for &j in &recovered {
2912            let seat_key = format!("judge-{}", j + 1);
2913            let spec = self.roles.judges[j].clone();
2914            let seat = self.seat(&seat_key, &spec.id);
2915            let mut text = prompt::final_vote(&labels, &language);
2916            if !has_context(&spec, &seat, sessions) {
2917                text = format!(
2918                    "{}\n\n# Candidates\n\n{}",
2919                    text,
2920                    self.candidate_block(&candidates, &base_short)
2921                );
2922            }
2923            vote_jobs.push(SeatJob {
2924                spec,
2925                seat,
2926                prompt: text,
2927                cwd: root.join(seat_key),
2928                timeout,
2929                allow_write: false,
2930                sessions,
2931                artifacts: artifacts.clone(),
2932                stem: format!("vote-judge-{}-recover", j + 1),
2933            });
2934            vote_pos.push(j);
2935        }
2936        let allowed = labels.clone();
2937        let mut vote_losses = Vec::new();
2938        let vote_retries = self.state.config.graph.retries;
2939        let vote_cache = self.state.config.cache_dir();
2940        let ctx = WaveCtx {
2941            run: &run_id,
2942            node: "vote",
2943            prompts: &prompts,
2944            cache: vote_cache.as_deref(),
2945            round: None,
2946        };
2947        let votes = ask_json_wave::<FinalVote>(
2948            vote_jobs,
2949            Arc::clone(&self.sem),
2950            vote_retries,
2951            &ctx,
2952            &mut vote_losses,
2953            &mut self.state,
2954            &move |v: &FinalVote| match v.label() {
2955                Some(c) if allowed.contains(&c) => Ok(()),
2956                other => bail!("vote {other:?} is not one of {allowed:?}"),
2957            },
2958        )
2959        .await;
2960        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
2961            let agent_id = seat.agent.clone();
2962            self.state.seats.insert(seat.key.clone(), seat);
2963            match res {
2964                Ok((v, _)) => {
2965                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2966                        rec.vote = v.label();
2967                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2968                    } else {
2969                        self.state.votes.push(VoteRecord {
2970                            judge: j + 1,
2971                            agent: agent_id,
2972                            vote: v.label(),
2973                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2974                            changed: false,
2975                        });
2976                    }
2977                    self.state.event(
2978                        "recover",
2979                        format!("judge {} voted again after the limit", j + 1),
2980                    );
2981                }
2982                Err(e) => {
2983                    self.state
2984                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2985                }
2986            }
2987        }
2988
2989        // A seat that ranked again is present even if its re-vote failed —
2990        // `tally` falls back to the initial ranking's first choice — so clear
2991        // its quota loss. Seats that still fail keep theirs and stay absent.
2992        let recovered_keys: BTreeSet<String> = recovered
2993            .iter()
2994            .map(|&j| format!("judge-{}", j + 1))
2995            .collect();
2996        self.state
2997            .quota
2998            .retain(|q| !recovered_keys.contains(&q.seat));
2999        // A seat that hit the limit again is a fresh loss, not the old one:
3000        // replace the stale entry so the history stays one-per-seat and the
3001        // daemon can tell this attempt's loss from a previous session's.
3002        for loss in judge_losses.into_iter().chain(vote_losses) {
3003            if recovered_keys.contains(&loss.seat) {
3004                continue;
3005            }
3006            self.state.quota.retain(|q| q.seat != loss.seat);
3007            self.state.quota.push(loss);
3008        }
3009
3010        // Recompute the verdict from the refreshed panel.
3011        self.state.tally = None;
3012        self.tally()?;
3013        Ok(self
3014            .state
3015            .tally
3016            .as_ref()
3017            .map(|t| t.met_quorum)
3018            .unwrap_or(false))
3019    }
3020
3021    // ----------------------------------------------------------------- fold
3022
3023    async fn fold_losers(&mut self) -> Result<()> {
3024        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3025            return Ok(());
3026        };
3027        let repo = self.state.repo.clone();
3028        let mut folded = Vec::new();
3029        for i in 0..self.state.candidates.len() {
3030            let c = &self.state.candidates[i];
3031            if c.label == winner || c.folded {
3032                continue;
3033            }
3034            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3035            git::worktree_remove(&repo, &wt).await.ok();
3036            git::branch_delete(&repo, &branch).await.ok();
3037            self.state.candidates[i].folded = true;
3038            folded.push(label.to_string());
3039        }
3040        // The judges are finished; their checkouts are pure cost from here.
3041        let root = self.state.worktree_root();
3042        for j in 1..=self.roles.judges.len() {
3043            let wt = root.join(format!("judge-{j}"));
3044            if wt.exists() {
3045                git::worktree_remove(&repo, &wt).await.ok();
3046            }
3047        }
3048        // The design-deliberation stage is finished by the time a tally
3049        // exists — same reasoning as the judges above.
3050        if self.state.config.graph.advise {
3051            for k in 1..=self.state.config.graph.advisors {
3052                let wt = root.join(format!("advisor-{k}"));
3053                if wt.exists() {
3054                    git::worktree_remove(&repo, &wt).await.ok();
3055                }
3056            }
3057        }
3058        if !folded.is_empty() {
3059            self.state
3060                .event("fold", format!("folded candidates {}", folded.join(", ")));
3061            self.state.save()?;
3062        }
3063        Ok(())
3064    }
3065
3066    // ------------------------------------------------------------ base sync
3067
3068    /// Land the winner's tree on the current tip of `<remote>/<base>` before
3069    /// anything verifies it.
3070    ///
3071    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
3072    /// read whatever is checked out in the winner's worktree. Left alone that
3073    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
3074    /// it when the run *branched* - and a run takes long enough that the base
3075    /// has usually moved by the time it gets here. A gate that ran there
3076    /// answers "green on the commit this run started from", not "green on
3077    /// what is about to land", and the difference showed up three times in
3078    /// one day as a green run whose merge would have reverted a file another
3079    /// pull request had already landed.
3080    ///
3081    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
3082    /// implementation of the same idea: `land::Step::Rebase` already worked
3083    /// out the rules - throwaway worktree, conflict stops and reports rather
3084    /// than feeding a fixer, nothing runs in the primary tree - and a second
3085    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
3086    /// warns about ("two answers to a question nobody notices until a diff is
3087    /// wrong").
3088    ///
3089    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
3090    /// so it survives a park/resume. A conflict or a push failure sets
3091    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
3092    /// as they were - untouched, for a person to look at - which is also what
3093    /// makes re-entering this function afterwards a no-op instead of a second
3094    /// attempt at the same wall.
3095    async fn sync_to_base(&mut self) -> Result<()> {
3096        if self
3097            .state
3098            .base_sync
3099            .as_ref()
3100            .is_some_and(|s| s.conflict.is_some())
3101        {
3102            return Ok(());
3103        }
3104        let Some(winner) = self.state.winner().cloned() else {
3105            return Ok(());
3106        };
3107
3108        let repo = self.state.repo.clone();
3109        let remote = self.state.config.merge.remote.clone();
3110        let base_branch = self.state.base_branch.clone();
3111        let tracking = format!("{remote}/{base_branch}");
3112
3113        git::fetch(&repo, &remote, &base_branch).await.ok();
3114        // No network, or the remote never had this branch: `resolve_base`
3115        // already treats that as non-fatal at branch time, and a run that got
3116        // this far must not be blocked by it here either.
3117        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3118            return Ok(());
3119        };
3120
3121        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3122        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3123        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3124
3125        if behind == 0 {
3126            self.state.base_sync = Some(BaseSync {
3127                tip,
3128                behind: 0,
3129                attempts,
3130                conflict: None,
3131            });
3132            self.state.save()?;
3133            return Ok(());
3134        }
3135
3136        if attempts >= BASE_SYNC_ROUNDS {
3137            let why = format!(
3138                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3139                 rebase(s); rebasing again would only race it",
3140                winner.branch
3141            );
3142            self.state.status = RunStatus::Blocked;
3143            self.state.base_sync = Some(BaseSync {
3144                tip,
3145                behind,
3146                attempts,
3147                conflict: Some(why.clone()),
3148            });
3149            self.state.event("land", why);
3150            self.state.save()?;
3151            return Ok(());
3152        }
3153
3154        self.state.event(
3155            "land",
3156            format!(
3157                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3158                winner.branch
3159            ),
3160        );
3161        self.state.save()?;
3162
3163        let scratch = self.state.dir().join("base-sync");
3164        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
3165        let attempts = attempts + 1;
3166        match rebased {
3167            Ok(None) => {
3168                // The branch ref moved, but a worktree that already had it
3169                // checked out (the winner's) was not told; sync its index and
3170                // files before anything reads them.
3171                git::sync_to_head(&winner.worktree).await?;
3172                self.state.base_sync = Some(BaseSync {
3173                    tip: tip.clone(),
3174                    behind: 0,
3175                    attempts,
3176                    conflict: None,
3177                });
3178                self.state
3179                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
3180            }
3181            Ok(Some(conflict)) => {
3182                let why = format!(
3183                    "{} conflicts with {tracking} and did not rebase: {}",
3184                    winner.branch,
3185                    conflict.chars().take(600).collect::<String>()
3186                );
3187                self.state.status = RunStatus::Blocked;
3188                self.state.base_sync = Some(BaseSync {
3189                    tip,
3190                    behind,
3191                    attempts,
3192                    conflict: Some(why.clone()),
3193                });
3194                self.state.event("land", why);
3195            }
3196            Err(e) => {
3197                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3198                self.state.status = RunStatus::Blocked;
3199                self.state.base_sync = Some(BaseSync {
3200                    tip,
3201                    behind,
3202                    attempts,
3203                    conflict: Some(why.clone()),
3204                });
3205                self.state.event("land", why);
3206            }
3207        }
3208        self.state.save()?;
3209        Ok(())
3210    }
3211
3212    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
3213    /// last landed the winner on, once it has run, else the commit the run
3214    /// branched from.
3215    ///
3216    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
3217    /// and `vote` all happen before there is a winner to rebase, so they
3218    /// compare every candidate against the branch point on purpose, and a
3219    /// base that moves after they are already done cannot change an answer
3220    /// they already gave.
3221    fn landing_base(&self) -> String {
3222        self.state
3223            .base_sync
3224            .as_ref()
3225            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3226    }
3227
3228    // ------------------------------------------------------- operator fix
3229
3230    /// Route specific, already-recorded review findings to a fixer for a
3231    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
3232    /// entry point.
3233    ///
3234    /// Distinct from `review_loop`'s own fix step in three ways: it never
3235    /// runs a reviewer wave, it never spends review-round budget, and what
3236    /// happened is recorded as an [`OperatorFixRequest`] appended to
3237    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
3238    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
3239    /// and vote must never be rewritten to look like a manufactured blocking
3240    /// verdict.
3241    ///
3242    /// Only meaningful once review has actually concluded: `Ready` (handed
3243    /// off with findings still open, or simply concluded clean while minor
3244    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
3245    /// gate failed). Everything else is refused: a run still in progress
3246    /// should simply be resumed, and a `Merged` run's branch has already
3247    /// landed — reopening *this* run's own record cannot change that, so the
3248    /// answer there is a fresh `magi review <branch>`.
3249    ///
3250    /// A real commit here re-verifies through a fresh, ordinary review-only
3251    /// run on the same branch ([`Self::review`]) rather than reopening this
3252    /// run's own `review_loop`: once any round in this run's history went
3253    /// clean, `review_conclusion` treats that as permanent by design (the
3254    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
3255    /// way to force one more genuine reviewer wave out of *this* run without
3256    /// either rewriting history or weakening that guarantee for every other
3257    /// caller. A review-only run costs nothing extra — no implementation, no
3258    /// judging, no vote — and exercises the exact same review → verify →
3259    /// gate → (human) merge path, unmodified.
3260    pub async fn fix_selected(
3261        &mut self,
3262        ids: &[String],
3263        reason: &str,
3264        allow_stale: bool,
3265    ) -> Result<()> {
3266        let reason = reason.trim();
3267        if reason.is_empty() {
3268            bail!("a fix request needs a reason — that is the operator's own record of why");
3269        }
3270        if ids.is_empty() {
3271            bail!("no finding id given");
3272        }
3273        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3274            bail!(
3275                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3276                 has already concluded — can be given a targeted fix. A run still \
3277                 in progress should simply be resumed; a `merged` run's branch has \
3278                 already landed, so its answer is a fresh `magi review <branch>`, \
3279                 not reopening this run's own record",
3280                self.state.id,
3281                self.state.status.as_str()
3282            );
3283        }
3284        let Some(winner) = self.state.winner().cloned() else {
3285            bail!("run {} has no winning candidate to fix", self.state.id);
3286        };
3287        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3288            bail!(
3289                "branch `{}` no longer exists; this run cannot be extended",
3290                winner.branch
3291            );
3292        }
3293        let home = crate::run::home();
3294        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3295            bail!(
3296                "run {} is currently being worked on by another magi process",
3297                self.state.id
3298            );
3299        }
3300        // Held for the rest of this call, including the follow-up review
3301        // below: two `magi fix` invocations against the same run must not
3302        // both reach the worktree manipulation further down, which would
3303        // otherwise race to remove and recreate the same directory — see
3304        // [`FixClaim`]'s own doc.
3305        let _claim = FixClaim::acquire(&self.state.dir())?;
3306
3307        // Resolve every id before spending anything — an unknown id refuses
3308        // the whole request rather than silently dropping it — and dedup
3309        // while keeping the operator's own order.
3310        let mut seen = BTreeSet::new();
3311        let mut findings = Vec::new();
3312        let mut missing = Vec::new();
3313        for id in ids {
3314            if !seen.insert(id.clone()) {
3315                continue;
3316            }
3317            match self.state.finding(id) {
3318                Some((round, rec, f)) => findings.push(OperatorFixFinding {
3319                    id: f.id.clone(),
3320                    severity: f.severity,
3321                    reviewer_vote: rec.vote,
3322                    round: round.round,
3323                    round_head: round.head.clone(),
3324                    reviewer: rec.reviewer,
3325                    agent: rec.agent.clone(),
3326                    file: f.file.clone(),
3327                    line: f.line,
3328                    title: f.title.clone(),
3329                    detail: f.detail.clone(),
3330                    outcome: OperatorFixOutcome::Pending,
3331                }),
3332                None => missing.push(id.clone()),
3333            }
3334        }
3335        if !missing.is_empty() {
3336            bail!(
3337                "unknown finding id(s): {}; nothing was changed",
3338                missing.join(", ")
3339            );
3340        }
3341
3342        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3343        let stale_details: Vec<(String, String)> = findings
3344            .iter()
3345            .filter(|f| f.round_head != head_at_request)
3346            .map(|f| (f.id.clone(), f.round_head.clone()))
3347            .collect();
3348        let stale = !stale_details.is_empty();
3349        if stale && !allow_stale {
3350            bail!(
3351                "the branch has moved since some finding(s) were raised — {} — now \
3352                 at {}; pass --allow-stale to fix anyway, or re-run review first",
3353                stale_details
3354                    .iter()
3355                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3356                    .collect::<Vec<_>>()
3357                    .join(", "),
3358                short(&head_at_request)
3359            );
3360        }
3361
3362        let request = OperatorFixRequest {
3363            requested_at: Timestamp::now(),
3364            reason: reason.to_owned(),
3365            findings,
3366            head_at_request: head_at_request.clone(),
3367            allow_stale,
3368            stale,
3369            fix: None,
3370            result_head: None,
3371            follow_up_review_run: None,
3372        };
3373        self.state.event(
3374            "fix",
3375            format!(
3376                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3377                request.findings.len(),
3378                request
3379                    .findings
3380                    .iter()
3381                    .map(|f| f.id.as_str())
3382                    .collect::<Vec<_>>()
3383                    .join(", "),
3384            ),
3385        );
3386        // Recorded now, before any worktree work or the fixer call itself —
3387        // and re-saved at each checkpoint below: a crash at any point after
3388        // this (mid fixer call, mid follow-up review) must not lose the fact
3389        // that this was requested, for which findings, and why. Everything
3390        // past this point reads and writes through `request_index` rather
3391        // than a local variable, since `request` itself is moved here.
3392        self.state.operator_fixes.push(request);
3393        self.state.save()?;
3394        let request_index = self.state.operator_fixes.len() - 1;
3395
3396        // A fresh, dedicated worktree for this one call, never the winner's
3397        // own worktree in place: that one may already be gone (folded away),
3398        // and reusing it in place would leave the branch checked out there
3399        // when the follow-up review below tries to check it out again. Freed
3400        // immediately after, either way — but only once confirmed clean:
3401        // `worktree_remove` is a `git worktree remove --force`, which would
3402        // otherwise discard uncommitted work left there by the operator or
3403        // another process before this had a chance to even look at it.
3404        if winner.worktree.exists() {
3405            // Lockfiles a rescue commit withheld stay untracked on purpose and
3406            // are already recorded; they are not the operator's work to protect.
3407            let dirty = git::git(
3408                &winner.worktree,
3409                &["status", "--porcelain", "--untracked-files=all"],
3410            )
3411            .await?;
3412            let only_withheld = dirty.lines().all(|l| {
3413                l.strip_prefix("?? ")
3414                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3415            });
3416            if !only_withheld {
3417                bail!(
3418                    "`{}` has uncommitted changes; refusing to touch it — commit or \
3419                     discard them first",
3420                    winner.worktree.display()
3421                );
3422            }
3423            git::worktree_remove(&self.state.repo, &winner.worktree)
3424                .await
3425                .ok();
3426        }
3427        let fix_worktree = self.state.worktree_root().join("operator-fix");
3428        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3429        git::git(
3430            &self.state.repo,
3431            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3432        )
3433        .await
3434        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3435        if !git::is_clean(&fix_worktree).await? {
3436            git::worktree_remove(&self.state.repo, &fix_worktree)
3437                .await
3438                .ok();
3439            bail!(
3440                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3441                winner.branch
3442            );
3443        }
3444
3445        let run_id = self.state.id.clone();
3446        let prompts = self.state.config.prompts.clone();
3447        let language = self.state.config.graph.language.clone();
3448        let sessions = self.state.config.graph.sessions;
3449        let artifacts = agent::artifacts_dir(&self.state.dir());
3450        let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3451            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3452            _ => (
3453                self.state
3454                    .config
3455                    .agent(&winner.agent)
3456                    .cloned()
3457                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3458                format!("impl-{}", winner.label),
3459            ),
3460        };
3461        let seat = self.seat(&fix_seat_key, &fix_spec.id);
3462        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3463            .findings
3464            .iter()
3465            .map(|f| Finding {
3466                id: f.id.clone(),
3467                severity: f.severity,
3468                file: f.file.clone(),
3469                line: f.line,
3470                title: f.title.clone(),
3471                detail: f.detail.clone(),
3472            })
3473            .collect();
3474        let job = SeatJob {
3475            prompt: prompt::operator_fix(
3476                &self.state.instruction,
3477                &finding_list,
3478                reason,
3479                &stale_details,
3480                &head_at_request,
3481                &language,
3482            ),
3483            spec: fix_spec.clone(),
3484            seat,
3485            cwd: fix_worktree.clone(),
3486            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3487            allow_write: true,
3488            sessions,
3489            artifacts: artifacts.clone(),
3490            stem: "operator-fix".to_owned(),
3491        };
3492        let cache = self.state.config.cache_dir();
3493        let ctx = WaveCtx {
3494            run: &run_id,
3495            node: "fix",
3496            prompts: &prompts,
3497            cache: cache.as_deref(),
3498            round: None,
3499        };
3500        let (seat, out) =
3501            run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3502        let agent_id = seat.agent.clone();
3503
3504        let mut fix = FixRecord {
3505            agent: agent_id,
3506            addressed: Vec::new(),
3507            rejected: Vec::new(),
3508            notes: String::new(),
3509            committed: false,
3510            failed: None,
3511            duration_ms: 0,
3512            continuation: None,
3513        };
3514        let mut final_seat = seat.clone();
3515        match out {
3516            AgentOutcome::Ok(o) => {
3517                fix.duration_ms = o.duration_ms;
3518                let parsed = verdict::extract_json::<FixReport>(&o.text);
3519                let incomplete_reason = match &parsed {
3520                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3521                        "the reply parsed, but it reported a command whose own CLI \
3522                         never confirmed an exit status"
3523                            .to_owned(),
3524                    ),
3525                    Ok(_) => None,
3526                    Err(e) => Some(e.to_string()),
3527                };
3528                match incomplete_reason {
3529                    None => {
3530                        let report = parsed.expect("checked Ok above");
3531                        fix.addressed = report.addressed;
3532                        fix.rejected = report.rejected;
3533                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
3534                    }
3535                    Some(reason) => {
3536                        let (resumed_seat, resolved, failure, cont) = self
3537                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
3538                            .await;
3539                        fix.duration_ms += cont.cumulative_wait_ms;
3540                        fix.continuation = Some(cont);
3541                        final_seat = resumed_seat;
3542                        match resolved {
3543                            Some(report) => {
3544                                fix.addressed = report.addressed;
3545                                fix.rejected = report.rejected;
3546                                fix.notes =
3547                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
3548                            }
3549                            None => fix.failed = failure,
3550                        }
3551                    }
3552                }
3553            }
3554            AgentOutcome::Dropped(o) => {
3555                fix.duration_ms = o.duration_ms;
3556                let why = o
3557                    .dropped
3558                    .as_ref()
3559                    .map(|d| d.why.as_str())
3560                    .unwrap_or("the CLI ended the stream without delivering its answer");
3561                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3562            }
3563            AgentOutcome::Quota(o) => {
3564                self.state.quota.push(QuotaLoss {
3565                    seat: final_seat.key.clone(),
3566                    node: "fix".to_owned(),
3567                    at: Timestamp::now(),
3568                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3569                });
3570                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3571            }
3572            AgentOutcome::Failed(e) => fix.failed = Some(e),
3573        }
3574        if fix.continuation.is_none() {
3575            fix.continuation = Some(ContinuationRecord::not_needed());
3576        }
3577        self.state.seats.insert(final_seat.key.clone(), final_seat);
3578
3579        let rescue_message = format!(
3580            "magi: operator-selected fix ({}) (uncommitted work)",
3581            self.state.operator_fixes[request_index]
3582                .findings
3583                .iter()
3584                .map(|f| f.id.as_str())
3585                .collect::<Vec<_>>()
3586                .join(", ")
3587        );
3588        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
3589            self.state.note_withheld("fix", &r.withheld);
3590        }
3591        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
3592        fix.committed = after != head_at_request;
3593        git::worktree_remove(&self.state.repo, &fix_worktree)
3594            .await
3595            .ok();
3596
3597        self.state.event(
3598            "fix",
3599            match &fix.failed {
3600                Some(reason) => format!(
3601                    "operator fix: adoption report was lost ({reason}); {}",
3602                    if fix.committed {
3603                        "committed"
3604                    } else {
3605                        "NO new commit"
3606                    }
3607                ),
3608                None => format!(
3609                    "operator fix: {} addressed, {} rejected, {}",
3610                    fix.addressed.len(),
3611                    fix.rejected.len(),
3612                    if fix.committed {
3613                        "committed"
3614                    } else {
3615                        "NO new commit"
3616                    }
3617                ),
3618            },
3619        );
3620
3621        // Every selected finding gets an outcome — never left `Pending` once
3622        // the fixer's own turn is over. A report that never came back at all
3623        // marks every one of them `Unreported`, not silently "not addressed":
3624        // quota, a dropped stream, or an exhausted continuation are gaps in
3625        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
3626        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
3627        for f in &mut self.state.operator_fixes[request_index].findings {
3628            f.outcome = if fix.failed.is_some() {
3629                OperatorFixOutcome::Unreported
3630            } else if fix.addressed.contains(&f.id) {
3631                OperatorFixOutcome::Addressed
3632            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
3633                OperatorFixOutcome::Rejected { why: r.why.clone() }
3634            } else {
3635                OperatorFixOutcome::Unreported
3636            };
3637        }
3638
3639        let committed = fix.committed;
3640        if committed {
3641            self.state.operator_fixes[request_index].result_head = Some(after.clone());
3642        }
3643        self.state.operator_fixes[request_index].fix = Some(fix);
3644        // Saved again now that the fixer's own outcome is final, on top of
3645        // the save right after the request was first pushed above.
3646        self.state.save()?;
3647
3648        if committed {
3649            self.state.event(
3650                "fix",
3651                format!(
3652                    "operator fix committed {}; opening a follow-up review-only run",
3653                    short(&after)
3654                ),
3655            );
3656            match Self::review(&self.state.repo, &winner.branch, self.state.config.clone()).await {
3657                Ok(mut follow_up) => {
3658                    follow_up.state.event(
3659                        "start",
3660                        format!(
3661                            "requested by an operator fix on run {} for finding(s) {}",
3662                            self.state.id,
3663                            self.state.operator_fixes[request_index]
3664                                .findings
3665                                .iter()
3666                                .map(|f| f.id.as_str())
3667                                .collect::<Vec<_>>()
3668                                .join(", "),
3669                        ),
3670                    );
3671                    follow_up.state.save()?;
3672                    let follow_up_id = follow_up.state.id.clone();
3673                    if let Err(e) = follow_up.execute().await {
3674                        self.state.event(
3675                            "fix",
3676                            format!(
3677                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
3678                            ),
3679                        );
3680                    }
3681                    self.state.operator_fixes[request_index].follow_up_review_run =
3682                        Some(follow_up_id);
3683                }
3684                Err(e) => {
3685                    self.state.event(
3686                        "fix",
3687                        format!("committed the fix but could not open a follow-up review: {e:#}"),
3688                    );
3689                }
3690            }
3691            self.state.save()?;
3692        }
3693
3694        Ok(())
3695    }
3696
3697    // --------------------------------------------------------------- review
3698
3699    /// The agent and seat key that fix the winner's tree: the configured
3700    /// fixer, else the winner's own implementer seat, whose conversation
3701    /// continues now that the competition is over. Shared by the review loop
3702    /// and the gate-fix round so both talk to the same seat.
3703    fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
3704        match &self.roles.fixer {
3705            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3706            _ => (
3707                self.state
3708                    .config
3709                    .agent(&winner.agent)
3710                    .cloned()
3711                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3712                format!("impl-{}", winner.label),
3713            ),
3714        }
3715    }
3716
3717    async fn review_loop(&mut self) -> Result<()> {
3718        // A base that would not rebase is a person's decision, not a review
3719        // round: nothing here would change the answer, and reviewers and a
3720        // fixer would be spending real budget on a tree that cannot land
3721        // regardless of what they find.
3722        if self
3723            .state
3724            .base_sync
3725            .as_ref()
3726            .is_some_and(|s| s.conflict.is_some())
3727        {
3728            return Ok(());
3729        }
3730        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3731        // agent files with `magi task add` name the run that paid for it. The
3732        // prompt overlay is cloned alongside it because the waves borrow it
3733        // while `self` is mutably borrowed by the node's own bookkeeping.
3734        let run_id = self.state.id.clone();
3735        let prompts = self.state.config.prompts.clone();
3736        let Some(winner) = self.state.winner().cloned() else {
3737            return Ok(());
3738        };
3739        let max_rounds = self.state.config.graph.review_rounds;
3740        // A clean round, an exhausted round budget, or a stalled tree (see
3741        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
3742        // they are recorded — recomputed here, not read off `status`, so a
3743        // reentry into a run that already stopped restates the identical
3744        // verdict instead of silently handing back whatever an earlier node
3745        // in this same walk clobbered `status` to (a solo-candidate
3746        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
3747        // below runs an empty range once the budget is spent, and would
3748        // otherwise fall through without touching `status` at all.
3749        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
3750            self.state.status = status;
3751            self.state.save()?;
3752            return Ok(());
3753        }
3754        self.state.status = RunStatus::Reviewing;
3755        // A last recorded round whose own verification never resolved
3756        // (`ResourceBlocked` — the shared build cache, not the patch) is
3757        // never a concluded round, whatever the round budget says: starting
3758        // a fresh round on top of it would spend a whole new reviewer wave
3759        // re-reading an unchanged patch instead of just retrying the one
3760        // check that actually needs it, and once the budget is spent the
3761        // loop below has nothing left to do at all (its range is empty).
3762        // Retry that check directly instead, exactly the same retry
3763        // `stop_reviewing` already does for its own catch-up case.
3764        if self
3765            .state
3766            .reviews
3767            .last()
3768            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
3769        {
3770            let shell = self.state.config.shell();
3771            return self
3772                .stop_reviewing(
3773                    "the last round's own verification never resolved",
3774                    &shell,
3775                    &winner.worktree,
3776                )
3777                .await;
3778        }
3779
3780        let repo = self.state.repo.clone();
3781        let root = self.state.worktree_root();
3782        let language = self.state.config.graph.language.clone();
3783        let sessions = self.state.config.graph.sessions;
3784        let artifacts = agent::artifacts_dir(&self.state.dir());
3785        let base = self.landing_base();
3786        let base_short = short(&base);
3787        let reviewers = self.roles.reviewers.clone();
3788        let shell = self.state.config.shell();
3789
3790        for round in (self.state.reviews.len() + 1)..=max_rounds {
3791            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3792            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
3793            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
3794            // The prior round's own record, already persisted — never a
3795            // hand-carried variable of just its failing output: that is
3796            // exactly what let a round's e2e result drift out of sync with
3797            // which commit it was actually about (see `SCHEMA`'s doc for
3798            // schema 8). Judged against `head`, the commit reviewers are
3799            // about to look at now, so the summary always reads as "an
3800            // earlier head" here — this round's own patch has not been
3801            // checked yet.
3802            let prev_verification = self
3803                .state
3804                .reviews
3805                .last()
3806                .and_then(|r| r.verification_summary(&head));
3807
3808            // Each reviewer gets its own detached checkout of exactly this
3809            // commit: nobody can perturb the winner's tree, and the fixer can
3810            // keep working without racing a reviewer.
3811            let mut jobs = Vec::new();
3812            for (r, spec) in reviewers.iter().cloned().enumerate() {
3813                let wt = root.join(format!("review-{}", r + 1));
3814                if wt.exists() {
3815                    git::reset_detached(&wt, &head).await?;
3816                } else {
3817                    git::worktree_add_detached(&repo, &wt, &head).await?;
3818                }
3819                let seat_key = format!("review-{}", r + 1);
3820                let seat = self.seat(&seat_key, &spec.id);
3821                jobs.push(SeatJob {
3822                    prompt: prompt::review(&prompt::ReviewCtx {
3823                        instruction: &self.state.instruction,
3824                        branch: &winner.branch,
3825                        base_short: &base_short,
3826                        stat: &stat,
3827                        patch: &patch,
3828                        verification: prev_verification.as_ref(),
3829                        reviewers: reviewers.len(),
3830                        round,
3831                        rounds: max_rounds,
3832                        // A review-only run has no rankings, so nothing
3833                        // competed for this patch and the reviewer is told so.
3834                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
3835                        lens: Lens::for_seat(r),
3836                        language: &language,
3837                    }),
3838                    spec,
3839                    seat,
3840                    cwd: wt,
3841                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3842                    allow_write: false,
3843                    sessions,
3844                    artifacts: artifacts.clone(),
3845                    stem: format!("review-{round}-{}", r + 1),
3846                });
3847            }
3848
3849            self.state.event(
3850                "review",
3851                format!(
3852                    "round {round}: {} reviewers on {}",
3853                    jobs.len(),
3854                    short(&head)
3855                ),
3856            );
3857            let mut quota_losses = Vec::new();
3858            let review_retries = self.state.config.graph.retries;
3859            let review_cache = self.state.config.cache_dir();
3860            let ctx = WaveCtx {
3861                run: &run_id,
3862                node: "review",
3863                prompts: &prompts,
3864                cache: review_cache.as_deref(),
3865                round: Some(round),
3866            };
3867            let results = ask_json_wave::<Review>(
3868                jobs,
3869                Arc::clone(&self.sem),
3870                review_retries,
3871                &ctx,
3872                &mut quota_losses,
3873                &mut self.state,
3874                &|_: &Review| Ok(()),
3875            )
3876            .await;
3877            // Counted before the move below: how many of *this* round's
3878            // reviewer seats were lost to their own rate limit, as opposed to
3879            // a crash, a timeout, or unparsable output — see `round_is_clean`.
3880            let round_quota_missing = quota_losses.len();
3881            self.state.quota.extend(quota_losses);
3882
3883            let mut records = Vec::new();
3884            let mut all_findings = Vec::new();
3885            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
3886                let agent_id = seat.agent.clone();
3887                self.state.seats.insert(seat.key.clone(), seat);
3888                let mut record = ReviewRecord {
3889                    reviewer: r + 1,
3890                    agent: agent_id,
3891                    summary: String::new(),
3892                    findings: Vec::new(),
3893                    vote: None,
3894                    failed: None,
3895                    duration_ms: 0,
3896                    // Set for both outcomes: `failed: Some(_)` with
3897                    // `attempts > 0` is a seat every retry still lost, not a
3898                    // recovered one — only `failed: None` with `attempts > 0`
3899                    // reads as "answered after a nudge" (see this field's own
3900                    // doc).
3901                    attempts,
3902                };
3903                match res {
3904                    Ok((review, out)) => {
3905                        // Sanitized here, at the point every other piece of
3906                        // agent prose in this file is (candidate summaries,
3907                        // deliberation turns, vote reasons): a reviewer's own
3908                        // words are the one thing about it that could name
3909                        // it, and reconsideration below broadcasts this same
3910                        // summary and these same findings to every other
3911                        // seat on the panel.
3912                        record.summary =
3913                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
3914                        record.vote = Some(review.vote);
3915                        record.duration_ms = out.duration_ms;
3916                        for (n, mut f) in review.findings.into_iter().enumerate() {
3917                            // ids are magi's, never the agent's: the fixer's
3918                            // adoption report is keyed by them.
3919                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
3920                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
3921                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
3922                            // `file` is agent-supplied prose too, never
3923                            // checked against the real tree — the same
3924                            // exposure `title`/`detail` above have, just in
3925                            // a field easy to forget because it looks like a
3926                            // path rather than free text.
3927                            f.file = f
3928                                .file
3929                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
3930                            all_findings.push(f.clone());
3931                            record.findings.push(f);
3932                        }
3933                        self.state.event(
3934                            "review",
3935                            format!(
3936                                "round {round}: reviewer {} voted {} with {} finding(s)",
3937                                r + 1,
3938                                review.vote.label(),
3939                                record.findings.len()
3940                            ),
3941                        );
3942                    }
3943                    Err(e) => {
3944                        record.failed = Some(e.to_string());
3945                        self.state.event(
3946                            "review",
3947                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
3948                        );
3949                    }
3950                }
3951                records.push(record);
3952            }
3953
3954            // Tally the round's votes and, if they split, spend the one
3955            // round of reconsideration the split -> deliberate -> revote
3956            // shape `judge`/`vote` use for the panel, sized down to what a
3957            // read-only review round can afford: one round, and a revote
3958            // rather than an argument, because the panel already wrote its
3959            // reasoning down as findings the first time around.
3960            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
3961            let vote_split =
3962                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
3963            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
3964            if vote_split {
3965                self.state.event(
3966                    "review",
3967                    format!(
3968                        "round {round}: votes split ({}) — one round of reconsideration",
3969                        initial_votes
3970                            .iter()
3971                            .map(|v| v.label())
3972                            .collect::<Vec<_>>()
3973                            .join(", ")
3974                    ),
3975                );
3976                // Seats read every seat's findings and votes, still numbered
3977                // and never named — the same anonymity `review` itself keeps.
3978                let panel: Vec<ReviewSeatReport<'_>> = records
3979                    .iter()
3980                    .filter_map(|r| {
3981                        r.vote.map(|vote| ReviewSeatReport {
3982                            reviewer: r.reviewer,
3983                            vote,
3984                            summary: &r.summary,
3985                            findings: &r.findings,
3986                        })
3987                    })
3988                    .collect();
3989
3990                let mut jobs = Vec::new();
3991                let mut seats_at = Vec::new();
3992                for (r, spec) in reviewers.iter().cloned().enumerate() {
3993                    // A seat with no initial vote has nothing to reconsider
3994                    // from and stays absent, the same as it stayed absent
3995                    // from `panel` above.
3996                    if records[r].vote.is_none() {
3997                        continue;
3998                    }
3999                    let wt = root.join(format!("review-{}", r + 1));
4000                    let seat_key = format!("review-{}", r + 1);
4001                    let seat = self.seat(&seat_key, &spec.id);
4002                    // A seat with no live session has already forgotten the
4003                    // initial review's prompt — restate the patch it is
4004                    // voting on, the same as `deliberate`/`vote` do for a
4005                    // judge in the same position.
4006                    let patch_ctx = if has_context(&spec, &seat, sessions) {
4007                        None
4008                    } else {
4009                        Some(ReviewPatch {
4010                            branch: &winner.branch,
4011                            base_short: &base_short,
4012                            stat: &stat,
4013                            patch: &patch,
4014                        })
4015                    };
4016                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
4017                        instruction: &self.state.instruction,
4018                        reviewer: r + 1,
4019                        lens: Lens::for_seat(r),
4020                        panel: &panel,
4021                        patch: patch_ctx,
4022                        round,
4023                        rounds: max_rounds,
4024                        language: &language,
4025                    });
4026                    jobs.push(SeatJob {
4027                        prompt,
4028                        spec,
4029                        seat,
4030                        cwd: wt,
4031                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4032                        allow_write: false,
4033                        sessions,
4034                        artifacts: artifacts.clone(),
4035                        stem: format!("review-{round}-reconsider-{}", r + 1),
4036                    });
4037                    seats_at.push(r);
4038                }
4039
4040                let mut recon_quota_losses = Vec::new();
4041                let recon_cache = self.state.config.cache_dir();
4042                let recon_ctx = WaveCtx {
4043                    run: &run_id,
4044                    node: "review",
4045                    prompts: &prompts,
4046                    cache: recon_cache.as_deref(),
4047                    round: Some(round),
4048                };
4049                let recon_results = ask_json_wave::<ReviewRevote>(
4050                    jobs,
4051                    Arc::clone(&self.sem),
4052                    review_retries,
4053                    &recon_ctx,
4054                    &mut recon_quota_losses,
4055                    &mut self.state,
4056                    &|_: &ReviewRevote| Ok(()),
4057                )
4058                .await;
4059                self.state.quota.extend(recon_quota_losses);
4060
4061                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4062                    let agent_id = seat.agent.clone();
4063                    self.state.seats.insert(seat.key.clone(), seat);
4064                    let mut rec = ReviewRevoteRecord {
4065                        reviewer: r + 1,
4066                        agent: agent_id,
4067                        vote: None,
4068                        reason: String::new(),
4069                        failed: None,
4070                    };
4071                    match res {
4072                        Ok((rv, _)) => {
4073                            rec.vote = Some(rv.vote);
4074                            rec.reason =
4075                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4076                            self.state.event(
4077                                "review",
4078                                format!(
4079                                    "round {round}: reviewer {} revoted {}",
4080                                    r + 1,
4081                                    rv.vote.label()
4082                                ),
4083                            );
4084                        }
4085                        Err(e) => {
4086                            rec.failed = Some(e.to_string());
4087                            self.state.event(
4088                                "review",
4089                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4090                            );
4091                        }
4092                    }
4093                    reconsideration.push(rec);
4094                }
4095            } else if initial_votes.len() > 1 {
4096                self.state.event(
4097                    "review",
4098                    format!(
4099                        "round {round}: votes agreed ({}) — no reconsideration",
4100                        initial_votes[0].label()
4101                    ),
4102                );
4103            }
4104
4105            // The final vote per seat is its revote where reconsideration
4106            // ran and answered, its initial vote otherwise — the same
4107            // fallback `tally` uses for a judge whose private vote failed.
4108            let final_votes: Vec<ReviewVote> = records
4109                .iter()
4110                .filter_map(|r| {
4111                    reconsideration
4112                        .iter()
4113                        .find(|rv| rv.reviewer == r.reviewer)
4114                        .and_then(|rv| rv.vote)
4115                        .or(r.vote)
4116                })
4117                .collect();
4118            let round_verdict = ReviewVote::worst(final_votes);
4119
4120            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4121            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4122            // A round that already has a blocking finding and a round left to
4123            // try is going back to the fixer no matter what `verify.e2e`
4124            // says, so running it first only spends the loop's slowest step
4125            // (minutes, for a Rust repo's full test suite) on a head about
4126            // to be rewritten. Deferred, never skipped: `verify.e2e` still
4127            // runs once a round has no blocking findings left (see
4128            // `round_is_clean`, which a deferred — empty — `e2e` can never
4129            // satisfy since `blocking` is nonzero whenever this branch is
4130            // taken), and `stop_reviewing` forces a real run before it will
4131            // ever read a deferred round as green.
4132            let defer_e2e =
4133                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4134            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4135                let reason =
4136                    format!("{blocking} blocking finding(s) already required a fix this round");
4137                self.state.event(
4138                    "verify",
4139                    format!(
4140                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4141                         {}); it will run once a round has none left",
4142                        short(&head)
4143                    ),
4144                );
4145                (Vec::new(), false, true, Some(reason))
4146            } else {
4147                let e2e_commands = self.state.config.verify.e2e.clone();
4148                let cache_dir = self.state.config.cache_dir();
4149                let context = format!("round {round}");
4150                let (e2e, verify_retried) = with_cache_lease(
4151                    &mut self.state,
4152                    cache_dir.as_deref(),
4153                    "e2e",
4154                    "e2e",
4155                    &winner.worktree,
4156                    &head,
4157                    verify_timeout,
4158                    &context,
4159                    |state, budget| {
4160                        let shell = shell.clone();
4161                        let e2e_commands = e2e_commands.clone();
4162                        let worktree = winner.worktree.clone();
4163                        let context = context.clone();
4164                        async move {
4165                            run_e2e_with_retry(
4166                                state,
4167                                &shell,
4168                                &e2e_commands,
4169                                &worktree,
4170                                budget,
4171                                &context,
4172                            )
4173                            .await
4174                        }
4175                    },
4176                )
4177                .await;
4178                (e2e, verify_retried, false, None)
4179            };
4180
4181            let expected = records.len();
4182            let answered = records.iter().filter(|r| r.failed.is_none()).count();
4183            let incomplete = answered < expected;
4184            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4185            let policy = self.state.config.graph.incomplete_review;
4186            let clean = round_is_clean(
4187                blocking,
4188                e2e_ok,
4189                answered,
4190                expected,
4191                round_quota_missing,
4192                policy,
4193            );
4194
4195            let mut round_record = ReviewRound {
4196                round,
4197                head: head.clone(),
4198                verified_head: None,
4199                verified_at: None,
4200                reviews: records,
4201                e2e,
4202                verify_retried,
4203                e2e_deferred,
4204                e2e_defer_reason,
4205                fix: None,
4206                blocking,
4207                answered,
4208                expected,
4209                clean,
4210                progressed: false,
4211                vote_split,
4212                reconsideration,
4213                verdict: round_verdict,
4214            };
4215            // Which commit and when magi actually attempted to check —
4216            // known the moment a command was dispatched against `head`,
4217            // whether or not it finished: a resource-blocked attempt still
4218            // targeted a specific commit at a specific time, and leaving
4219            // that unrecorded is exactly what made `verification_summary`
4220            // report a fresh attempt as "commit unknown ... recorded before
4221            // this was tracked", indistinguishable from a genuinely old,
4222            // untracked record. Only a deferred or unconfigured round never
4223            // ran at all and has nothing to record — see
4224            // `ReviewRound::verified_head`'s own doc.
4225            if !matches!(
4226                round_record.e2e_status(),
4227                E2eStatus::Deferred | E2eStatus::NotConfigured
4228            ) {
4229                round_record.verified_head = Some(head.clone());
4230                round_record.verified_at = Some(Timestamp::now());
4231            }
4232            let this_round_verification = round_record.verification_summary(&head);
4233
4234            if incomplete {
4235                let missing: Vec<String> = round_record
4236                    .reviews
4237                    .iter()
4238                    .filter(|r| r.failed.is_some())
4239                    .map(|r| format!("review-{}", r.reviewer))
4240                    .collect();
4241                self.state.event(
4242                    "review",
4243                    format!(
4244                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4245                        missing.join(", ")
4246                    ),
4247                );
4248            }
4249
4250            if clean {
4251                self.state.event(
4252                    "review",
4253                    if incomplete && policy == IncompleteReviewPolicy::Warn {
4254                        format!(
4255                            "round {round}: clean (warn policy, incomplete panel) — no \
4256                             blocking findings from the seats that answered, verification green"
4257                        )
4258                    } else if incomplete {
4259                        format!(
4260                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4261                             quorum) — no blocking findings from the seats that answered, \
4262                             verification green",
4263                            expected - answered
4264                        )
4265                    } else {
4266                        format!("round {round}: clean — no blocking findings, verification green")
4267                    },
4268                );
4269                self.state.reviews.push(round_record);
4270                self.state.status = RunStatus::Gating;
4271                self.state.save()?;
4272                return Ok(());
4273            }
4274
4275            // Nothing was raised and verification passed, but not every seat
4276            // answered and `round_is_clean` still refused to call it clean —
4277            // either a seat is missing for a reason other than its own quota
4278            // (a crash, a timeout, unparsable output — worth another try), or
4279            // every seat that could have answered lost its quota and nobody
4280            // is left to decide on: re-review rather than send the fixer
4281            // after a round with nothing to fix.
4282            if incomplete && blocking == 0 && e2e_ok {
4283                self.state.reviews.push(round_record);
4284                self.state.save()?;
4285                if round == max_rounds {
4286                    self.state.status = RunStatus::Blocked;
4287                    self.state.event(
4288                        "review",
4289                        format!(
4290                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4291                             refusing to call it clean",
4292                            expected - answered
4293                        ),
4294                    );
4295                    return Ok(());
4296                }
4297                continue;
4298            }
4299
4300            // Nothing for the fixer to act on (`blocking == 0`) and the only
4301            // reason this round is not clean is that magi itself never got
4302            // a command to run — the shared build cache, not the patch (see
4303            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
4304            // the fixer would invite a change to appease contention that has
4305            // nothing to do with the diff, and would leave this attempt
4306            // sitting in the next round's prompt as if it were about an
4307            // earlier, superseded commit rather than what it actually is:
4308            // the same head, still waiting to be checked. Wait for it the
4309            // same way the final round's own contention is already handled,
4310            // whatever round this happens to be.
4311            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4312                self.state.reviews.push(round_record);
4313                return self
4314                    .stop_reviewing(
4315                        "the round's own verification could not run",
4316                        &shell,
4317                        &winner.worktree,
4318                    )
4319                    .await;
4320            }
4321
4322            if round == max_rounds {
4323                self.state.reviews.push(round_record);
4324                return self
4325                    .stop_reviewing(
4326                        &format!(
4327                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4328                        ),
4329                        &shell,
4330                        &winner.worktree,
4331                    )
4332                    .await;
4333            }
4334
4335            // Fix. The winner's own implementer seat continues its conversation:
4336            // the competition is over, so context is pure benefit now.
4337            let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4338            let seat = self.seat(&fix_seat_key, &fix_spec.id);
4339            let blocking_findings: Vec<_> = all_findings
4340                .iter()
4341                .filter(|f| f.severity.blocks())
4342                .cloned()
4343                .collect();
4344            let job = SeatJob {
4345                prompt: prompt::fix(
4346                    &self.state.instruction,
4347                    &blocking_findings,
4348                    this_round_verification.as_ref(),
4349                    round,
4350                    max_rounds,
4351                    &language,
4352                ),
4353                spec: fix_spec.clone(),
4354                seat,
4355                cwd: winner.worktree.clone(),
4356                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4357                allow_write: true,
4358                sessions,
4359                artifacts: artifacts.clone(),
4360                stem: format!("fix-{round}"),
4361            };
4362            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4363            let cache = self.state.config.cache_dir();
4364            let ctx = WaveCtx {
4365                run: &run_id,
4366                node: "fix",
4367                prompts: &prompts,
4368                cache: cache.as_deref(),
4369                round: Some(round),
4370            };
4371            let (seat, out) =
4372                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4373            let agent_id = seat.agent.clone();
4374
4375            let mut fix = FixRecord {
4376                agent: agent_id,
4377                addressed: Vec::new(),
4378                rejected: Vec::new(),
4379                notes: String::new(),
4380                committed: false,
4381                failed: None,
4382                duration_ms: 0,
4383                continuation: None,
4384            };
4385            let mut continuation = ContinuationRecord::not_needed();
4386            let mut final_seat = seat.clone();
4387            match out {
4388                AgentOutcome::Ok(o) => {
4389                    fix.duration_ms = o.duration_ms;
4390                    let parsed = verdict::extract_json::<FixReport>(&o.text);
4391                    // A parsed report standing next to a command this same
4392                    // reply's own CLI never confirmed the exit status of is
4393                    // not a resolved answer — the identical `CommandEvidence`
4394                    // `state.jobs` renders, read here instead of only on
4395                    // display, per the completion judgment and the shown
4396                    // record needing to agree.
4397                    let incomplete_reason = match &parsed {
4398                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4399                            "the reply parsed, but it reported a command whose own CLI never \
4400                             confirmed an exit status"
4401                                .to_owned(),
4402                        ),
4403                        Ok(_) => None,
4404                        Err(e) => Some(e.to_string()),
4405                    };
4406                    match incomplete_reason {
4407                        None => {
4408                            let report = parsed.expect("checked Ok above");
4409                            fix.addressed = report.addressed;
4410                            fix.rejected = report.rejected;
4411                            fix.notes =
4412                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
4413                        }
4414                        Some(reason) => {
4415                            let (resumed_seat, resolved, failure, cont) = self
4416                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4417                                .await;
4418                            fix.duration_ms += cont.cumulative_wait_ms;
4419                            continuation = cont;
4420                            final_seat = resumed_seat;
4421                            match resolved {
4422                                Some(report) => {
4423                                    fix.addressed = report.addressed;
4424                                    fix.rejected = report.rejected;
4425                                    fix.notes = blind::sanitize_prose(
4426                                        &report.notes,
4427                                        &self.state.config.blind,
4428                                    );
4429                                }
4430                                None => fix.failed = failure,
4431                            }
4432                        }
4433                    }
4434                }
4435                // The CLI's raw error JSON is not a fix report to parse.
4436                AgentOutcome::Dropped(o) => {
4437                    fix.duration_ms = o.duration_ms;
4438                    let why = o
4439                        .dropped
4440                        .as_ref()
4441                        .map(|d| d.why.as_str())
4442                        .unwrap_or("the CLI ended the stream without delivering its answer");
4443                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4444                }
4445                AgentOutcome::Quota(o) => {
4446                    self.state.quota.push(QuotaLoss {
4447                        seat: final_seat.key.clone(),
4448                        node: "fix".to_owned(),
4449                        at: Timestamp::now(),
4450                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4451                    });
4452                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4453                }
4454                AgentOutcome::Failed(e) => fix.failed = Some(e),
4455            }
4456            fix.continuation = Some(continuation);
4457            self.state.seats.insert(final_seat.key.clone(), final_seat);
4458            if let Ok(r) = git::rescue_commit(
4459                &winner.worktree,
4460                &format!("magi: review round {round} fixes (uncommitted work)"),
4461            )
4462            .await
4463            {
4464                self.state.note_withheld("fix", &r.withheld);
4465            }
4466            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
4467            fix.committed = after != before;
4468            // Judged by what `git` says moved against base, never by the
4469            // fixer's own `addressed`/`rejected` count — see
4470            // `ReviewRound::progressed`. Propagated with `?`, the same as the
4471            // `patch` snapshot above: swallowing this error would default
4472            // `diff_after` to empty, which almost always differs from a
4473            // non-empty `patch` and reads as "progressed" — exactly backwards
4474            // for a `git` failure the stagnation check cannot see through.
4475            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
4476            let progressed = diff_after != patch;
4477            let commit_note = if fix.committed {
4478                "committed"
4479            } else {
4480                "NO new commit"
4481            };
4482            let tree_note = if progressed {
4483                "changed vs base"
4484            } else {
4485                "unchanged vs base"
4486            };
4487            self.state.event(
4488                "fix",
4489                match &fix.failed {
4490                    // Distinct on purpose from "0 addressed, 0 rejected": the
4491                    // fixer's own diff still landed (blocking counts do keep
4492                    // falling round over round), only its adoption report did
4493                    // not come back, so this must never read like every
4494                    // finding was reviewed and declined.
4495                    Some(reason) => {
4496                        format!(
4497                            "round {round}: fixer's adoption report was lost ({reason}); \
4498                             {commit_note}, tree {tree_note}"
4499                        )
4500                    }
4501                    None => format!(
4502                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
4503                         {tree_note}{}",
4504                        fix.addressed.len(),
4505                        fix.rejected.len(),
4506                        if continuation.outcome == ContinuationOutcome::Resumed {
4507                            format!(
4508                                " (adoption report recovered after {} continuation(s))",
4509                                continuation.attempts
4510                            )
4511                        } else {
4512                            String::new()
4513                        },
4514                    ),
4515                },
4516            );
4517            round_record.fix = Some(fix);
4518            round_record.progressed = progressed;
4519            self.state.reviews.push(round_record);
4520            self.state.save()?;
4521
4522            // The fixer's own report never came back this round, even after
4523            // `continue_fix_report`'s own budget was spent on it — not an
4524            // ordinary "no report" (dropped stream, quota, plain failure),
4525            // which already reads that way and is left to the existing round
4526            // budget. Stopping here, rather than opening another round, is
4527            // what keeps a next reviewer/fixer wave from ever being
4528            // dispatched onto `winner.worktree` while whatever the seat's
4529            // last call may still have running there is unaccounted for: no
4530            // process liveness check exists (and none is being added — see
4531            // AGENTS.md/this task's own scope), so the only way to honour
4532            // "nothing starts before a valid report returns" is to not start
4533            // anything further on this worktree from this run at all.
4534            if matches!(
4535                continuation.outcome,
4536                ContinuationOutcome::Exhausted
4537                    | ContinuationOutcome::QuotaLost
4538                    | ContinuationOutcome::NoSession
4539            ) {
4540                return self
4541                    .stop_reviewing(
4542                        "the fixer's adoption report never came back, even after resuming its \
4543                         own seat; refusing to start another round against the same worktree \
4544                         while that is unresolved",
4545                        &shell,
4546                        &winner.worktree,
4547                    )
4548                    .await;
4549            }
4550
4551            let streak = self
4552                .state
4553                .reviews
4554                .iter()
4555                .rev()
4556                .take_while(|r| !r.progressed)
4557                .count();
4558            if streak >= STAGNANT_LIMIT {
4559                return self
4560                    .stop_reviewing(
4561                        &format!(
4562                            "the tree has not moved against base for {streak} round(s) in a row"
4563                        ),
4564                        &shell,
4565                        &winner.worktree,
4566                    )
4567                    .await;
4568            }
4569        }
4570        Ok(())
4571    }
4572
4573    /// Decide, from the last recorded round's own verification, whether
4574    /// stopping the review loop is a hand-off or a genuine block.
4575    ///
4576    /// Called once the loop has given up trying — the round budget is spent,
4577    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
4578    /// findings still open, never while a round is still clean or the
4579    /// incomplete-panel case handled inline above. Gate and e2e are facts
4580    /// about the tree; a lingering review finding is an opinion, and this
4581    /// workload's own `magi stats` puts reviewer precision low enough
4582    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
4583    /// must not by itself stand between a green, verified change and the
4584    /// human who decides what to do with it. A red e2e is not an opinion, so
4585    /// that case still blocks, with the failing command and a tail of its
4586    /// output recorded here rather than left in `run.json` for someone to go
4587    /// find.
4588    ///
4589    /// A round that deferred its own e2e (see [`Config::graph`]'s
4590    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
4591    /// only because nothing ran, and treating an empty list as a passing one
4592    /// here is exactly the "deferred painted green" bug this function exists
4593    /// to not have. When the last round's own verification never resolved —
4594    /// deferred on purpose, or a real attempt the shared build cache blocked
4595    /// — this makes (or retries) the real run, on the actual worktree this
4596    /// loop is about to stop touching, before deciding anything. A
4597    /// resource-blocked attempt is likewise never read as either green or
4598    /// red: it is evidence about the machine, not the patch (see
4599    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
4600    /// blocked cache leaves this call without deciding rather than guessing
4601    /// — the caller retries on a later reentry.
4602    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
4603        let round_idx = self.state.reviews.len() - 1;
4604        // A deferred round and a resource-blocked one are the same shape
4605        // here: neither has a real result yet, and both get one more
4606        // attempt. Read off `e2e_status` — the single source for this —
4607        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
4608        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
4609        // still retries instead of being read as a settled result the
4610        // instant it stops being empty.
4611        let needs_catchup_run = matches!(
4612            self.state.reviews[round_idx].e2e_status(),
4613            E2eStatus::Deferred | E2eStatus::ResourceBlocked
4614        );
4615        if needs_catchup_run {
4616            let round = self.state.reviews[round_idx].round;
4617            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4618            let commands = self.state.config.verify.e2e.clone();
4619            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
4620            let cache_dir = self.state.config.cache_dir();
4621            let context = format!(
4622                "round {round}: verification unresolved, catching up before the final decision"
4623            );
4624            let (outcomes, verify_retried) = with_cache_lease(
4625                &mut self.state,
4626                cache_dir.as_deref(),
4627                "e2e",
4628                "e2e",
4629                worktree,
4630                &attempted_head,
4631                timeout,
4632                &context,
4633                |state, budget| {
4634                    let shell = shell.to_vec();
4635                    let commands = commands.clone();
4636                    let context = context.clone();
4637                    async move {
4638                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
4639                            .await
4640                    }
4641                },
4642            )
4643            .await;
4644            let last = &mut self.state.reviews[round_idx];
4645            last.e2e = outcomes;
4646            last.verify_retried = verify_retried;
4647            // Always the commit and time this attempt actually targeted,
4648            // whether or not it happens to equal the reviewed `head` and
4649            // whether or not a command finished — see
4650            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
4651            // attempt is recorded too, so a later reader sees "attempted
4652            // again at T2" rather than silence.
4653            last.verified_head = Some(attempted_head);
4654            last.verified_at = Some(Timestamp::now());
4655            if verify_inconclusive(&last.e2e) {
4656                // Still not a real result: `e2e_deferred` is left exactly
4657                // as it was, so `needs_catchup_run` above reads
4658                // `ResourceBlocked` (via `e2e_status`, which checks
4659                // `resource_blocked` before `e2e_deferred`) and retries
4660                // again on the next reentry, rather than recording
4661                // contention as a red e2e and blocking the run on it.
4662                self.state.save()?;
4663                return Ok(());
4664            }
4665            last.e2e_deferred = false;
4666        }
4667        let last = &self.state.reviews[round_idx];
4668        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
4669
4670        match last.e2e_status() {
4671            E2eStatus::Failed => {
4672                let red: Vec<String> = last
4673                    .e2e
4674                    .iter()
4675                    .filter(|o| !o.ok())
4676                    .map(|o| {
4677                        format!(
4678                            "`{}` -> {:?}\n{}",
4679                            o.command,
4680                            o.code,
4681                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4682                        )
4683                    })
4684                    .collect();
4685                self.state
4686                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
4687                self.state.status = RunStatus::Blocked;
4688            }
4689            // `needs_catchup_run` above already retried once this call; if
4690            // it is still blocked, this is magi's own admission it could
4691            // not get a command to run, never a verdict on the patch — the
4692            // run is left exactly where a later reentry can retry again.
4693            E2eStatus::ResourceBlocked => {
4694                self.state.event(
4695                    "review",
4696                    format!(
4697                        "{why}; e2e could not run (shared build cache unavailable); not \
4698                         deciding yet"
4699                    ),
4700                );
4701            }
4702            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
4703                self.state.event(
4704                    "review",
4705                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
4706                );
4707                self.state.status = RunStatus::Gating;
4708            }
4709        }
4710        self.state.save()?;
4711        Ok(())
4712    }
4713
4714    // ----------------------------------------------------------------- gate
4715
4716    async fn gate(&mut self) -> Result<()> {
4717        // Judged by the review record itself, not by `status`: a solo
4718        // candidate's `judge`/`deliberate` skip rewrites `status` on every
4719        // reentry (see `judge`), and trusting it here is exactly how a run
4720        // that exhausted its review budget got gated and merged a second
4721        // time around. `review_conclusion` recomputes the review loop's own
4722        // verdict from the round records themselves — `Gating` for a clean
4723        // round or a hand-off (see `stop_reviewing`), anything else means the
4724        // loop is still going or genuinely blocked.
4725        // A base the winner could not be replayed onto is a decision, not a
4726        // round: there is no landing tree to gate. Read as its own record for
4727        // the same reason the review verdict is.
4728        if self.state.status == RunStatus::Failed
4729            || self
4730                .state
4731                .base_sync
4732                .as_ref()
4733                .is_some_and(|s| s.conflict.is_some())
4734            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
4735                != Some(RunStatus::Gating)
4736        {
4737            return Ok(());
4738        }
4739        if self.state.gate_ran {
4740            // `review_loop` derives its conclusion from the clean review
4741            // record on every reentry and therefore puts a completed run back
4742            // in `Gating`. A recorded gate is a stronger, terminal fact:
4743            // retain its original command output (or lack of any, for a repo
4744            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
4745            // doc) and restore `Blocked` on a real failure rather than
4746            // pretending the command is still running or running it a second
4747            // time. `gate_ran == false` remains the only shape — unattempted,
4748            // or a resource-blocked retry — that may still need to execute a
4749            // command.
4750            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
4751                self.state.status = RunStatus::Blocked;
4752                self.state.save()?;
4753            }
4754            return Ok(());
4755        }
4756        let Some(winner) = self.state.winner().cloned() else {
4757            return Ok(());
4758        };
4759        self.state.status = RunStatus::Gating;
4760        let mut outcomes = self.run_gate(&winner).await?;
4761        loop {
4762            // A resource-blocked outcome means the gate command never actually
4763            // ran - the shared build cache could not be acquired or confirmed
4764            // fresh in time - which is evidence about the machine, not about
4765            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
4766            // Recording it as a red gate would mark a run `Blocked` on nothing
4767            // but contention magi has already logged; leaving `self.state.gate`
4768            // empty and `self.state.gate_ran` false instead keeps the shape
4769            // this function already treats as "still needs to run" (see the
4770            // early-return above), so the next call retries the command
4771            // rather than concluding anything.
4772            if verify_inconclusive(&outcomes) {
4773                self.state.save()?;
4774                return Ok(());
4775            }
4776            if outcomes.iter().all(CommandOutcome::ok) {
4777                break;
4778            }
4779            match self.gate_fix_round(&winner, &outcomes).await? {
4780                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
4781                GateFix::Stop => break,
4782                GateFix::Defer => {
4783                    self.state.save()?;
4784                    return Ok(());
4785                }
4786            }
4787        }
4788        let passed = outcomes.iter().all(CommandOutcome::ok);
4789        self.state.gate = outcomes;
4790        self.state.gate_ran = true;
4791        if !passed {
4792            self.state.status = RunStatus::Blocked;
4793            let spent = self.state.gate_fixes.len();
4794            self.state.event(
4795                "gate",
4796                if spent == 0 {
4797                    "gate failed; not merging".to_owned()
4798                } else {
4799                    format!("gate failed after {spent} gate-fix round(s); not merging")
4800                },
4801            );
4802        }
4803        self.state.save()?;
4804        Ok(())
4805    }
4806
4807    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
4808    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
4809    /// after review is clean and never on a candidate awaiting judging.
4810    ///
4811    /// Never fails the run: a non-zero exit or timeout is a warning and a
4812    /// recorded outcome, and the gate remains the single arbiter. Nothing
4813    /// configured means nothing happens - no event, no commit. `commit_all`
4814    /// commits any leftover change under the neutral identity and returns
4815    /// `false` when the tree is clean, so no empty commit is ever made.
4816    async fn run_pre_gate(&mut self, winner: &Candidate) {
4817        let commands = self.state.config.verify.pre_gate.clone();
4818        if commands.is_empty() {
4819            return;
4820        }
4821        let shell = self.state.config.shell();
4822        let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4823        let (outcomes, _) = run_commands(
4824            &mut self.state,
4825            "pre_gate",
4826            "pre_gate",
4827            0,
4828            &shell,
4829            &commands,
4830            &winner.worktree,
4831            timeout,
4832        )
4833        .await;
4834        for o in &outcomes {
4835            if !o.ok() {
4836                tracing::warn!(
4837                    "pre_gate `{}` failed ({:?}); the gate decides",
4838                    o.command,
4839                    o.code
4840                );
4841            }
4842            self.state.event(
4843                "pre_gate",
4844                format!(
4845                    "`{}` -> {}",
4846                    o.command,
4847                    if o.ok() {
4848                        "pass".to_owned()
4849                    } else {
4850                        format!(
4851                            "FAIL ({:?})\n{}",
4852                            o.code,
4853                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4854                        )
4855                    }
4856                ),
4857            );
4858        }
4859        self.state.pre_gate = outcomes;
4860        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
4861            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
4862                Ok(head) => {
4863                    self.state
4864                        .event("pre_gate", format!("committed mechanical fixes ({head})"));
4865                    self.state.pre_gate_commit = Some(head);
4866                }
4867                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
4868            },
4869            Ok(false) => {}
4870            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
4871        }
4872        if let Err(e) = self.state.save() {
4873            tracing::warn!("could not persist the pre_gate record: {e:#}");
4874        }
4875    }
4876
4877    /// Run `verify.gate` once against the winner's current tree, logging one
4878    /// event per command. Empty when nothing is configured.
4879    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
4880        self.run_pre_gate(winner).await;
4881        let shell = self.state.config.shell();
4882        let gate_commands = self.state.config.verify.gate.clone();
4883        // Zero commands has nothing to run and nothing that could touch the
4884        // shared build cache, so it never needs a lease: `Config::cache_dir`
4885        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
4886        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
4887        // otherwise queue behind an unrelated run's lease and come back
4888        // resource-blocked - `gate_ran` would stay false on nothing but
4889        // cache contention, for a step that had nothing to check in the
4890        // first place.
4891        let outcomes = if gate_commands.is_empty() {
4892            Vec::new()
4893        } else {
4894            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4895            let cache_dir = self.state.config.cache_dir();
4896            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4897            let (outcomes, _) = with_cache_lease(
4898                &mut self.state,
4899                cache_dir.as_deref(),
4900                "gate",
4901                "gate",
4902                &winner.worktree,
4903                &head,
4904                timeout,
4905                "final gate",
4906                |state, budget| {
4907                    let shell = shell.clone();
4908                    let gate_commands = gate_commands.clone();
4909                    let worktree = winner.worktree.clone();
4910                    async move {
4911                        let (outcomes, timed_out_pids) = run_commands(
4912                            state,
4913                            "gate",
4914                            "gate",
4915                            0,
4916                            &shell,
4917                            &gate_commands,
4918                            &worktree,
4919                            budget,
4920                        )
4921                        .await;
4922                        (outcomes, false, timed_out_pids)
4923                    }
4924                },
4925            )
4926            .await;
4927            outcomes
4928        };
4929        if outcomes.is_empty() {
4930            // Nothing configured to check — distinct from every other
4931            // silence in this run's event log, since an empty `gate` alone
4932            // no longer says whether the gate ran at all (see
4933            // `RunState::gate_ran`'s own doc).
4934            self.state.event(
4935                "gate",
4936                "no gate commands configured; nothing to check, passing",
4937            );
4938        }
4939        for o in &outcomes {
4940            self.state.event(
4941                "gate",
4942                format!(
4943                    "`{}` -> {}",
4944                    o.command,
4945                    if o.ok() {
4946                        "pass".to_owned()
4947                    } else {
4948                        format!(
4949                            "FAIL ({:?})\n{}",
4950                            o.code,
4951                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
4952                        )
4953                    }
4954                ),
4955            );
4956        }
4957        Ok(outcomes)
4958    }
4959
4960    /// One bounded fix round for a failing gate.
4961    ///
4962    /// The fixer is told the failure came from the gate itself, not from a
4963    /// reviewer, and is shown the failed commands, their exit codes and a tail
4964    /// of their output - whatever `[verify].gate` holds, nothing here knows
4965    /// what those commands run. Only a normal non-zero exit that printed
4966    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
4967    /// command or a full disk says nothing about the code, and a fixer sent
4968    /// after it can only appease the machine. The round is judged by what git
4969    /// says moved, never by the fixer's own report, and `verify.e2e` runs
4970    /// again before the gate does, so a fix cannot trade a green gate for a
4971    /// red e2e unnoticed.
4972    async fn gate_fix_round(
4973        &mut self,
4974        winner: &Candidate,
4975        outcomes: &[CommandOutcome],
4976    ) -> Result<GateFix> {
4977        let cap = self.state.config.graph.gate_fix_rounds;
4978        let spent = self.state.gate_fixes.len();
4979        if spent >= cap {
4980            if cap > 0 {
4981                self.state.event(
4982                    "gate",
4983                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
4984                );
4985            }
4986            return Ok(GateFix::Stop);
4987        }
4988        if !gate_fixable(outcomes) {
4989            self.state.event(
4990                "gate",
4991                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
4992                 command or similar); not spending a fix round on it",
4993            );
4994            return Ok(GateFix::Stop);
4995        }
4996        let min_free = self.state.config.disk.min_free_bytes;
4997        if min_free > 0 {
4998            match crate::disk::free_bytes(&winner.worktree) {
4999                Ok(free) if crate::disk::enough_space(free, min_free) => {}
5000                Ok(free) => {
5001                    self.state.event(
5002                        "gate",
5003                        format!(
5004                            "only {free} bytes free ({min_free} required by `[disk] \
5005                             min_free_bytes`); not spending a fix round on a failure the disk \
5006                             may explain"
5007                        ),
5008                    );
5009                    return Ok(GateFix::Stop);
5010                }
5011                Err(e) => {
5012                    self.state.event(
5013                        "gate",
5014                        format!("free disk space could not be measured ({e:#}); no fix round"),
5015                    );
5016                    return Ok(GateFix::Stop);
5017                }
5018            }
5019        }
5020
5021        let attempt = spent + 1;
5022        let run_id = self.state.id.clone();
5023        let prompts = self.state.config.prompts.clone();
5024        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5025        let base = self.landing_base();
5026        let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5027        let seat = self.seat(&fix_seat_key, &fix_spec.id);
5028        let job = SeatJob {
5029            prompt: prompt::gate_fix(
5030                &self.state.instruction,
5031                &failed,
5032                attempt,
5033                cap,
5034                &self.state.config.graph.language,
5035            ),
5036            spec: fix_spec,
5037            seat,
5038            cwd: winner.worktree.clone(),
5039            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5040            allow_write: true,
5041            sessions: self.state.config.graph.sessions,
5042            artifacts: agent::artifacts_dir(&self.state.dir()),
5043            stem: format!("gate-fix-{attempt}"),
5044        };
5045        self.state.event(
5046            "gate",
5047            format!("gate failed; gate-fix round {attempt} of {cap}"),
5048        );
5049        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5050        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5051        let cache = self.state.config.cache_dir();
5052        let ctx = WaveCtx {
5053            run: &run_id,
5054            node: "gate-fix",
5055            prompts: &prompts,
5056            cache: cache.as_deref(),
5057            round: None,
5058        };
5059        let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5060        let mut record = GateFixRecord {
5061            agent: seat.agent.clone(),
5062            failed,
5063            notes: String::new(),
5064            committed: false,
5065            error: None,
5066        };
5067        match out {
5068            AgentOutcome::Ok(o) => {
5069                // A missing report is not a failed fix: the round is judged
5070                // by the tree below, and the report only carries prose.
5071                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5072                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5073                }
5074            }
5075            AgentOutcome::Dropped(_) => {
5076                record.error = Some("the CLI dropped the stream".to_owned());
5077            }
5078            AgentOutcome::Quota(o) => {
5079                self.state.quota.push(QuotaLoss {
5080                    seat: seat.key.clone(),
5081                    node: "gate-fix".to_owned(),
5082                    at: Timestamp::now(),
5083                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5084                });
5085                record.error = Some("rate limited (quota); fixer could not run".to_owned());
5086            }
5087            AgentOutcome::Failed(e) => record.error = Some(e),
5088        }
5089        self.state.seats.insert(seat.key.clone(), seat);
5090        if let Ok(r) = git::rescue_commit(
5091            &winner.worktree,
5092            &format!("magi: gate fix {attempt} (uncommitted work)"),
5093        )
5094        .await
5095        {
5096            self.state.note_withheld("gate-fix", &r.withheld);
5097        }
5098        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5099        record.committed = after != before;
5100        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5101        let note = record.error.clone();
5102        self.state.gate_fixes.push(record);
5103        self.state.save()?;
5104        if !changed {
5105            self.state.event(
5106                "gate",
5107                match note {
5108                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5109                    None => format!("gate-fix round {attempt}: the tree did not change"),
5110                },
5111            );
5112            return Ok(GateFix::Stop);
5113        }
5114        self.state.event(
5115            "gate",
5116            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5117        );
5118
5119        let commands = self.state.config.verify.e2e.clone();
5120        if !commands.is_empty() {
5121            let shell = self.state.config.shell();
5122            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5123            let cache_dir = self.state.config.cache_dir();
5124            let context = format!("gate-fix round {attempt}");
5125            let (e2e, _) = with_cache_lease(
5126                &mut self.state,
5127                cache_dir.as_deref(),
5128                "e2e",
5129                "e2e",
5130                &winner.worktree,
5131                &after,
5132                timeout,
5133                &context,
5134                |state, budget| {
5135                    let shell = shell.clone();
5136                    let commands = commands.clone();
5137                    let context = context.clone();
5138                    let worktree = winner.worktree.clone();
5139                    async move {
5140                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5141                            .await
5142                    }
5143                },
5144            )
5145            .await;
5146            if verify_inconclusive(&e2e) {
5147                return Ok(GateFix::Defer);
5148            }
5149            if e2e.iter().any(|o| !o.ok()) {
5150                self.state.event(
5151                    "gate",
5152                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5153                );
5154                return Ok(GateFix::Stop);
5155            }
5156        }
5157        Ok(GateFix::Retry)
5158    }
5159
5160    // ---------------------------------------------------------------- merge
5161
5162    async fn merge(&mut self) -> Result<()> {
5163        // Same reasoning as `gate`: ask the review and gate records directly
5164        // rather than `status`, which a solo-candidate `judge`/`deliberate`
5165        // skip can rewrite on reentry to something that no longer says
5166        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
5167        // so a hand-off (open findings, green verification) reaches merge
5168        // exactly like a genuinely clean round does.
5169        //
5170        // A run resumed mid-`land` never reaches here at all: `execute`
5171        // recognises `RunStatus::Landing` before it even calls `prep`, and
5172        // routes straight to `run_land` instead. That has to happen a level
5173        // up from this function, not with a check in here, because
5174        // `review_loop`'s own status recomputation (see its doc) runs
5175        // *before* `merge` on every reentry and would otherwise overwrite
5176        // the `Landing` marker with `Gating` before this node ever saw it.
5177        if self
5178            .state
5179            .base_sync
5180            .as_ref()
5181            .is_some_and(|s| s.conflict.is_some())
5182            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5183                != Some(RunStatus::Gating)
5184            // `gate_ran == false` is not "passed" - `gate` leaves it false
5185            // both before it has ever run and when its last attempt was
5186            // resource-blocked (see `Runner::gate`'s own doc), and neither is
5187            // permission to merge on nothing but the review record. Only a
5188            // gate that actually ran - zero commands configured and
5189            // vacuously passed, or one or more that all exited 0 - may
5190            // proceed; `RunState::gate_status` is the single place that
5191            // reading is computed.
5192            || !self.state.gate_status().ok()
5193        {
5194            return Ok(());
5195        }
5196        // This node's own record, not `status`: `status == Ready` is not
5197        // unique to the harmless `MergeMode::None` path this line was
5198        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
5199        // run's PR was closed without merging — and on that run `mode` is
5200        // still `Pr`, so a reentry that fell through here would push and
5201        // open a second pull request. `self.state.merge` is set exactly once
5202        // this node (or `land`) has already produced a verdict, under every
5203        // mode, which is what "already done" actually means here.
5204        if self.state.merge.is_some() {
5205            return Ok(());
5206        }
5207        let Some(winner) = self.state.winner().cloned() else {
5208            return Ok(());
5209        };
5210        let repo = self.state.repo.clone();
5211        let base = self.state.base_branch.clone();
5212        let mode = self.state.config.merge.mode;
5213        let style = self.state.config.merge.style;
5214        let pr = pr_message(&self.state, winner.label);
5215        let message = pr.commit_message();
5216
5217        let outcome = match mode {
5218            MergeMode::None => MergeOutcome {
5219                mode,
5220                ok: true,
5221                detail: manual_merge_command(style, &repo, &winner.branch, &message),
5222            },
5223            MergeMode::Local => {
5224                let on = git::current_branch(&repo).await?;
5225                if on.as_deref() != Some(base.as_str()) {
5226                    MergeOutcome {
5227                        mode,
5228                        ok: false,
5229                        detail: format!(
5230                            "{} has {} checked out, not the base branch {base}",
5231                            repo.display(),
5232                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
5233                        ),
5234                    }
5235                } else if !git::is_clean(&repo).await? {
5236                    MergeOutcome {
5237                        mode,
5238                        ok: false,
5239                        detail: format!("{} is dirty; refusing to merge", repo.display()),
5240                    }
5241                } else {
5242                    let out = match style {
5243                        MergeStyle::Merge => {
5244                            git::merge_no_ff(&repo, &winner.branch, &message).await?
5245                        }
5246                        MergeStyle::Squash => {
5247                            git::merge_squash(&repo, &winner.branch, &message).await?
5248                        }
5249                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5250                    };
5251                    MergeOutcome {
5252                        mode,
5253                        ok: out.ok(),
5254                        detail: if out.ok() { out.stdout } else { out.stderr },
5255                    }
5256                }
5257            }
5258            MergeMode::Pr => {
5259                let remote = self.state.config.merge.remote.clone();
5260                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5261                if !pushed.ok() {
5262                    MergeOutcome {
5263                        mode,
5264                        ok: false,
5265                        detail: pushed.stderr,
5266                    }
5267                } else {
5268                    let out =
5269                        gh_pr_create(&winner.worktree, &base, &winner.branch, &pr.title, &pr.body)
5270                            .await;
5271                    match out {
5272                        Ok(url) => MergeOutcome {
5273                            mode,
5274                            ok: true,
5275                            detail: url,
5276                        },
5277                        Err(e) => MergeOutcome {
5278                            mode,
5279                            ok: false,
5280                            detail: e.to_string(),
5281                        },
5282                    }
5283                }
5284            }
5285        };
5286
5287        self.state.status = match (mode, outcome.ok) {
5288            (MergeMode::None, _) => RunStatus::Ready,
5289            (_, true) => RunStatus::Merged,
5290            (_, false) => RunStatus::Blocked,
5291        };
5292        self.state.event(
5293            "merge",
5294            format!(
5295                "{:?}: {}",
5296                mode,
5297                outcome.detail.lines().next().unwrap_or("")
5298            ),
5299        );
5300        self.state.merge = Some(outcome);
5301        self.state.save()?;
5302
5303        // The PR is open and the run would historically stop here, leaving the
5304        // operator to watch checks, feed review comments back to a fixer, and
5305        // merge. That was done by hand six times in one session before this
5306        // existed. Opt-in, because merging is the one irreversible thing magi
5307        // can do to a repository.
5308        if self.state.config.graph.land
5309            && mode == MergeMode::Pr
5310            && self.state.status == RunStatus::Merged
5311        {
5312            self.run_land().await?;
5313        }
5314        // `run_land` may have left `status` at `Landing` - still waiting on
5315        // CI or the owner's approval, not actually settled - so this has to
5316        // read whatever `status` ended up as here, not the `Merged` this
5317        // function set a few lines up.
5318        self.settle_questions();
5319        Ok(())
5320    }
5321
5322    /// Enter `land`.
5323    ///
5324    /// Shared between a fresh run's first pass through [`Runner::merge`] and
5325    /// a resumed run's re-entry. `land::land` itself is what serialises the
5326    /// two git-mutating moments inside the loop — the rebase push and
5327    /// `gh pr merge` — per repository (see its own doc); nothing here needs
5328    /// to hold a lock across the whole call, and doing so would serialise
5329    /// this run's CI wait against a *different* run's land-approval resume
5330    /// in the same repository, which is exactly the "must not wait on
5331    /// another task" property the daemon's slot-freeing exists to give.
5332    async fn run_land(&mut self) -> Result<()> {
5333        let url = self
5334            .state
5335            .merge
5336            .as_ref()
5337            .map(|m| m.detail.clone())
5338            .unwrap_or_default();
5339        let url = url.lines().next().unwrap_or("").trim().to_owned();
5340        if !url.starts_with("http") {
5341            return Ok(());
5342        }
5343        // A land failure is not a lost run: the work is on a branch and the
5344        // pull request is open, which is exactly where a human takes over.
5345        match land::land(&mut self.state, &url).await {
5346            Ok(pr) if self.state.parked => {
5347                // `land` already saved the parked marker; nothing here
5348                // overrides `status` back to a terminal value while an
5349                // approval is still outstanding.
5350                let _ = pr;
5351            }
5352            Ok(pr) => {
5353                self.state.status = match pr.state {
5354                    land::PrLifecycle::Merged => RunStatus::Merged,
5355                    _ => RunStatus::Blocked,
5356                };
5357                // Downstream of a confirmed merge only - see
5358                // `bump::should_release_bump`'s own doc for why this one
5359                // check covers all three of `land`'s success paths.
5360                // Best-effort: the run already landed, so a failure here
5361                // (the decision call, `gh`, `cargo`) is recorded and never
5362                // turns a landed run into a failed one.
5363                if bump::should_release_bump(self.state.status)
5364                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
5365                {
5366                    // Deliberately only an event: most `Err`s here mean the
5367                    // bump did not apply (no `Cargo.toml`, no agent
5368                    // installed, an unusable decision), not that a release
5369                    // PR is stranded. `after_merge` raises its own notice
5370                    // once a PR exists and needs a human.
5371                    self.state
5372                        .event("bump", format!("release bump skipped: {e:#}"));
5373                }
5374                self.state.save()?;
5375            }
5376            Err(e) => {
5377                self.state.status = RunStatus::Blocked;
5378                self.state.event("land", format!("gave up: {e}"));
5379                self.state.save()?;
5380            }
5381        }
5382        Ok(())
5383    }
5384
5385    // -------------------------------------------------------------- helpers
5386
5387    /// Fetch or create a seat, keeping its conversation across nodes.
5388    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
5389        if let Some(existing) = self.state.seats.get(key)
5390            && existing.agent == agent
5391        {
5392            return existing.clone();
5393        }
5394        let fresh = SeatState::new(key, agent, self.state.seed);
5395        self.state.seats.insert(key.to_owned(), fresh.clone());
5396        fresh
5397    }
5398
5399    /// A candidate rendered for judging, with the leak policy applied.
5400    fn view(&self, c: &Candidate) -> CandidateView {
5401        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
5402            .unwrap_or_default();
5403        let (patch, _) = blind::sanitize_patch(
5404            &format!("candidate {} patch", c.label),
5405            &raw,
5406            &self.state.config.blind,
5407        );
5408        CandidateView {
5409            label: c.label,
5410            branch: c.branch.clone(),
5411            summary: c.summary.clone(),
5412            stat: c.stat.clone(),
5413            patch,
5414        }
5415    }
5416
5417    /// The full candidate set as prompt text, for seats with no live session.
5418    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
5419        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
5420        prompt::judge(
5421            "(see above)",
5422            &views,
5423            self.roles.judges.len(),
5424            base_short,
5425            "en",
5426        )
5427    }
5428
5429    /// Anonymised transcript for judge `self_idx`.
5430    ///
5431    /// The initial rankings are always the opening statements. Seeding them
5432    /// only when no turn had been taken yet meant every judge after the first
5433    /// argued against a single voice instead of against the actual split — the
5434    /// disagreement is the information, so it is always on the table.
5435    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
5436        let mut turns = Vec::new();
5437        for j in &self.state.judgements {
5438            if j.ranking.is_empty() {
5439                continue;
5440            }
5441            let reasons = j
5442                .reasons
5443                .iter()
5444                .map(|(k, v)| format!("- {k}: {v}"))
5445                .collect::<Vec<_>>()
5446                .join("\n");
5447            turns.push(Turn {
5448                who: format!("Judge {} (opening ranking)", j.judge),
5449                is_self: j.judge == self_idx + 1,
5450                body: format!(
5451                    "Ranked {}{}{reasons}",
5452                    j.ranking.iter().collect::<String>(),
5453                    if reasons.is_empty() {
5454                        ""
5455                    } else {
5456                        ", because:\n"
5457                    }
5458                ),
5459            });
5460        }
5461        for t in self
5462            .state
5463            .deliberation
5464            .iter()
5465            .flat_map(|r| r.turns.iter())
5466            .chain(current)
5467        {
5468            turns.push(Turn {
5469                who: format!("Judge {}", t.judge),
5470                is_self: t.judge == self_idx + 1,
5471                body: t.body.clone(),
5472            });
5473        }
5474        turns
5475    }
5476}
5477
5478/// Does this seat still hold the context a follow-up prompt would rely on?
5479fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
5480    agent::has_session(spec.kind, seat, sessions)
5481}
5482
5483/// The next entry in `roster` after `start`, never wrapping back to the
5484/// front, whose id is not in `tried` yet.
5485///
5486/// Starts one past `start` rather than at the front of `roster`: `start` is
5487/// the seat's own original position, and a seat whose candidate slot already
5488/// sits on the roster's second entry must fall through to the third next, not
5489/// restart at the first — which is very likely a different candidate's own
5490/// agent already. Never wraps back past `start`, for the same reason: an
5491/// entry earlier in the roster than the seat's own position is almost
5492/// certainly some *other* candidate slot's own agent, and once the tail of
5493/// the roster is exhausted there are no more untried agents for *this* seat
5494/// to fall through to — the caller's fallback chain ends there, exactly as
5495/// "no further untried agents remain in the list for that seat" asks for.
5496///
5497/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
5498/// the same id twice (an operator's `roles.implementers` typo, or a
5499/// `[[agents]]` list reused across roles) must not let
5500/// [`Runner::resume_quota_losses`] retry that id forever — one forward pass
5501/// over `roster` either finds an untried id or runs out, so this always
5502/// terminates regardless of duplicates.
5503fn next_untried_implementer<'a>(
5504    roster: &'a [AgentSpec],
5505    start: usize,
5506    tried: &BTreeSet<String>,
5507) -> Option<&'a AgentSpec> {
5508    roster
5509        .get(start + 1..)?
5510        .iter()
5511        .find(|s| !tried.contains(&s.id))
5512}
5513
5514/// Did this reply report running a command whose own CLI never confirmed an
5515/// exit status?
5516///
5517/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
5518/// command *finished* (see that type's own doc), so this can only be `true`
5519/// for a command whose completion event carried no readable exit code — not
5520/// for one that simply is not mentioned at all. That is the one signal this
5521/// crate can read, from the same record `state.jobs` renders, about a reply
5522/// standing next to work its own CLI cannot vouch for finishing; it is
5523/// deliberately not a check on the exit code's *value* (a fixer legitimately
5524/// runs a command that fails mid-iteration before it succeeds) and not a
5525/// guess at a command still running in the background (which emits no event
5526/// at all, and so leaves no evidence here to find).
5527fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
5528    commands.iter().any(|c| c.exit_code.is_none())
5529}
5530
5531/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
5532/// trusted as a verified no-op — the adoption guard's own text-level half.
5533///
5534/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
5535/// timed out): a marker only earns the benefit of the doubt from a turn the
5536/// CLI itself vouches for finishing properly, the same house style
5537/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
5538/// *fix* report to for `commands`. A candidate that timed out, exited
5539/// non-zero, or left a command unconfirmed is read as the ordinary loss it
5540/// is, whatever prose it wrote — this returns `None` before it ever looks at
5541/// `text`. The remaining guards (the tree really is empty, the evidence is
5542/// non-empty) are the caller's: this only reads what the reply *claimed*.
5543fn verified_noop_claim(
5544    usable: bool,
5545    commands: &[agent::CommandEvidence],
5546    text: &str,
5547) -> Option<String> {
5548    (usable && !has_unconfirmed_command(commands))
5549        .then(|| verdict::verified_noop(text))
5550        .flatten()
5551}
5552
5553fn short(commit: &str) -> String {
5554    commit.chars().take(7).collect()
5555}
5556
5557fn make_executable(path: &Path) -> Result<()> {
5558    #[cfg(unix)]
5559    {
5560        use std::os::unix::fs::PermissionsExt as _;
5561        let mut perms = std::fs::metadata(path)?.permissions();
5562        perms.set_mode(0o755);
5563        std::fs::set_permissions(path, perms)?;
5564    }
5565    #[cfg(not(unix))]
5566    {
5567        let _ = path;
5568    }
5569    Ok(())
5570}
5571
5572/// What every seat in one batch shares: where the answers are attributed, the
5573/// prompt overlay they inherit, and the build cache they are told to use.
5574///
5575/// A struct rather than four more parameters: `wave` also needs the run's
5576/// state (to record who is answering right now) and the attempt number, and
5577/// eight positional arguments is both unreadable and a clippy error.
5578struct WaveCtx<'a> {
5579    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
5580    /// paid for it.
5581    run: &'a str,
5582    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
5583    node: &'a str,
5584    prompts: &'a Prompts,
5585    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
5586    cache: Option<&'a Path>,
5587    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
5588    /// `JobRecord::round`. `None` for every other node.
5589    round: Option<usize>,
5590}
5591
5592/// Run one job, honouring the parallelism budget.
5593async fn run_one(
5594    job: SeatJob,
5595    sem: Arc<Semaphore>,
5596    ctx: &WaveCtx<'_>,
5597    state: &mut RunState,
5598    attempt: usize,
5599) -> (SeatState, AgentOutcome) {
5600    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
5601        .await
5602        .pop()
5603        .expect("one job in, one result out");
5604    (seat, out)
5605}
5606
5607/// Run every job concurrently, capped by the semaphore, preserving order.
5608///
5609/// Every seat in the batch is recorded into [`RunState::active`] before the
5610/// wave starts and cleared as each answer lands, so the run's own record says
5611/// who is still being waited on rather than only who finished.
5612async fn wave(
5613    jobs: Vec<SeatJob>,
5614    sem: Arc<Semaphore>,
5615    ctx: &WaveCtx<'_>,
5616    state: &mut RunState,
5617    attempt: usize,
5618) -> Vec<(usize, SeatState, AgentOutcome)> {
5619    let WaveCtx {
5620        run,
5621        node,
5622        prompts,
5623        cache,
5624        round,
5625    } = *ctx;
5626    for job in &jobs {
5627        state.seat_started(node, &job.seat.key, job.timeout, attempt);
5628    }
5629    if let Err(e) = state.save() {
5630        // A failed persist of "who is answering right now" must not abort the
5631        // wave: the seats are already being asked, and the alternative is
5632        // losing the answers to save a status line nobody may even be
5633        // watching.
5634        tracing::warn!("could not persist in-progress seats: {e:#}");
5635    }
5636    // Hold the shared build cache's lease for the whole batch, not per job:
5637    // several candidates (an implement wave) or a fixer legitimately share
5638    // one cache concurrently within this run, and that stays untouched — a
5639    // single lease taken once for the whole wave and released once it is
5640    // done is what stops a *different* borrower (another run's own wave, its
5641    // e2e/gate, a human's `magi review`) from interleaving a build into the
5642    // same directory while this one is in flight. Best-effort, not
5643    // all-or-nothing: a wave that cannot get the lease within its own
5644    // longest job's budget still runs — an hour of paid implementer calls is
5645    // not thrown away over cache contention — but every write-allowed seat
5646    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
5647    // below), the same fallback a read-only seat always gets, rather than
5648    // building into a directory this run was never granted. The identity
5649    // record is still invalidated below either way, so the next tracked
5650    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
5651    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
5652    let wait_started = Instant::now();
5653    let cache_guard = if let Some(cache_dir) = cache {
5654        if jobs_had_a_writer {
5655            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
5656            let budget = jobs
5657                .iter()
5658                .map(|j| j.timeout)
5659                .max()
5660                .unwrap_or(Duration::from_secs(60));
5661            acquire_cache_lease(state, cache_dir, &owner, budget, node)
5662                .await
5663                .ok()
5664        } else {
5665            None
5666        }
5667    } else {
5668        None
5669    };
5670    // Carved out of each job's own budget, not added on top of it: a seat
5671    // that waited behind the lease must not also get its full timeout
5672    // afterward, or a run contended on the cache could double the time it
5673    // spends per wave. `saturating_sub` floors at zero rather than
5674    // wrapping - a job whose whole budget was spent waiting starts with
5675    // none left, which is the honest number, not a free minimum.
5676    let waited_for_lease = wait_started.elapsed();
5677    let mut set = tokio::task::JoinSet::new();
5678    let overlay = prompts.overlay(node);
5679    for (i, mut job) in jobs.into_iter().enumerate() {
5680        job.timeout = job.timeout.saturating_sub(waited_for_lease);
5681        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
5682        if cache.is_some() {
5683            job.prompt.push('\n');
5684            job.prompt
5685                .push_str(&prompt::build_cache_note(node, job.allow_write));
5686        }
5687        let sem = Arc::clone(&sem);
5688        let run = run.to_owned();
5689        let node = node.to_owned();
5690        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
5691        // `prompt::build_cache_note`'s doc for why setting it anyway is
5692        // exactly how a sandboxed reviewer's write refusal got reported as a
5693        // defect in the patch, not a property of its own seat. And a
5694        // write-allowed one is handed it only when the lease above was
5695        // actually acquired: a wave that could not get it (`cache_guard` is
5696        // `None`, see its own comment) must not send seats to build into a
5697        // directory this run does not hold - that is the exact concurrent,
5698        // unmanaged-write race this module exists to prevent, not something
5699        // "proceeding anyway" is allowed to reintroduce.
5700        let cache = cache
5701            .filter(|_| job.allow_write && cache_guard.is_some())
5702            .map(Path::to_path_buf);
5703        set.spawn(async move {
5704            let _permit = sem.acquire().await;
5705            let mut seat = job.seat;
5706            let out = agent::invoke(
5707                &job.spec,
5708                &mut seat,
5709                &Invocation {
5710                    cwd: &job.cwd,
5711                    prompt: &job.prompt,
5712                    timeout: job.timeout,
5713                    allow_write: job.allow_write,
5714                    sessions: job.sessions,
5715                    artifacts: &job.artifacts,
5716                    stem: &job.stem,
5717                    run: &run,
5718                    node: &node,
5719                    cache_dir: cache.as_deref(),
5720                    attachments: &[],
5721                },
5722            )
5723            .await;
5724            let out = match out {
5725                Ok(o) if o.usable() => AgentOutcome::Ok(o),
5726                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
5727                // Billed work the CLI failed to hand over is not an ordinary
5728                // failure, but its text is the CLI's raw error JSON, not an
5729                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
5730                // read it as one by forgetting to check. `usable()` is always
5731                // false here (dropped implies an empty response), so this has
5732                // to be checked before the catch-all `Failed` below or the
5733                // one shape this exists for is lost with the rest.
5734                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
5735                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
5736                Ok(o) => AgentOutcome::Failed(format!(
5737                    "exited with {:?} and no usable output",
5738                    o.exit_code
5739                )),
5740                Err(e) => AgentOutcome::Failed(e.to_string()),
5741            };
5742            (i, seat, out)
5743        });
5744    }
5745    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
5746    while let Some(joined) = set.join_next().await {
5747        let (i, seat, out) = match joined {
5748            Ok(v) => v,
5749            // No seat to clear: a panicked task never reported which one it
5750            // was. The defensive sweep below this loop is what stops that
5751            // seat's `active` entry from surviving forever.
5752            Err(e) => {
5753                tracing::error!("agent task panicked: {e}");
5754                continue;
5755            }
5756        };
5757        state.seat_finished(&seat.key);
5758        record_jobs(state, node, round, &seat.key, &out);
5759        if let Err(e) = state.save() {
5760            tracing::warn!("could not persist a seat's completion: {e:#}");
5761        }
5762        if collected.len() <= i {
5763            collected.resize_with(i + 1, || None);
5764        }
5765        collected[i] = Some((i, seat, out));
5766    }
5767    // Belt-and-braces for the panic branch above: every seat this exact batch
5768    // started shares this `(node, attempt)` pair, and every seat that finished
5769    // normally already cleared itself, so anything left tagged with it here
5770    // can only be a panicked task's leftover. Cleared unconditionally rather
5771    // than left to read as still answering forever.
5772    if state
5773        .active
5774        .values()
5775        .any(|a| a.node == node && a.attempt == attempt)
5776    {
5777        state
5778            .active
5779            .retain(|_, a| !(a.node == node && a.attempt == attempt));
5780        if let Err(e) = state.save() {
5781            tracing::warn!("could not persist the end of a wave: {e:#}");
5782        }
5783    }
5784    // Whether or not the lease above was actually held, several worktrees
5785    // may just have built into the cache with nothing here able to name one
5786    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
5787    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
5788    // clean it might not have strictly needed; trusting a stale match would
5789    // cost it a wrong answer.
5790    if let Some(cache_dir) = cache
5791        && jobs_had_a_writer
5792    {
5793        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
5794    }
5795    if let Some(guard) = cache_guard {
5796        guard.release();
5797    }
5798    collected.into_iter().flatten().collect()
5799}
5800
5801/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
5802/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
5803/// this is data collection, not the fix-specific completion contract in
5804/// [`Runner::continue_fix_report`], and applies regardless of which node
5805/// asked.
5806///
5807/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
5808/// evidence from; `Failed` does not, and correctly contributes nothing — a
5809/// timeout or crash is not itself evidence about a command the seat may have
5810/// started.
5811fn record_jobs(
5812    state: &mut RunState,
5813    node: &str,
5814    round: Option<usize>,
5815    seat: &str,
5816    out: &AgentOutcome,
5817) {
5818    let commands: &[agent::CommandEvidence] = match out {
5819        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
5820        AgentOutcome::Failed(_) => &[],
5821    };
5822    let checked_at = Timestamp::now();
5823    for c in commands {
5824        state.jobs.push(JobRecord {
5825            node: node.to_owned(),
5826            round,
5827            seat: seat.to_owned(),
5828            id: c.id.clone(),
5829            description: c.description.clone(),
5830            checked_at,
5831            status: match c.exit_code {
5832                Some(0) => JobStatus::Completed,
5833                Some(_) => JobStatus::Failed,
5834                None => JobStatus::Unknown,
5835            },
5836            exit_code: c.exit_code,
5837            result_summary: c.result_summary.clone(),
5838            source: c.source.clone(),
5839        });
5840    }
5841}
5842
5843/// Is a review round clean, given how many reviewer seats answered against
5844/// how many the round expected?
5845///
5846/// A seat that never answered (timeout, crash, unparsable output) is not a
5847/// seat that read the patch and found nothing — treating it as such is
5848/// exactly the bug this function exists to close. Under the default `block`
5849/// policy a missing seat can never be clean; `warn` still requires the seats
5850/// that *did* answer to have found nothing blocking and verification to be
5851/// green.
5852///
5853/// `quota_missing` narrows that `block` default for exactly one cause of
5854/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
5855/// a session limit lifts by the very next round buys nothing — the seat is
5856/// asked again with the same quota — so once every missing seat is accounted
5857/// for by a quota loss (and at least one seat *did* answer, so a decision has
5858/// something to rest on) the round is decided on the panel that could answer,
5859/// same as `warn` would. A panel that lost every seat to quota is not
5860/// decided here: `answered == 0` falls through to the existing `block`
5861/// fallback so a fully collapsed panel still waits rather than landing on no
5862/// review at all.
5863fn round_is_clean(
5864    blocking: usize,
5865    e2e_ok: bool,
5866    answered: usize,
5867    expected: usize,
5868    quota_missing: usize,
5869    policy: IncompleteReviewPolicy,
5870) -> bool {
5871    if blocking != 0 || !e2e_ok {
5872        return false;
5873    }
5874    if answered == expected || policy == IncompleteReviewPolicy::Warn {
5875        return true;
5876    }
5877    answered > 0 && expected - answered <= quota_missing
5878}
5879
5880/// The review loop's own conclusion, derived entirely from its persisted
5881/// round records and the round budget that produced them — never from
5882/// `status`, so a reentry (or `gate`/`merge` reading it independently)
5883/// recomputes the identical answer regardless of what an earlier node in the
5884/// same walk, or a previous walk, did to `status`.
5885///
5886/// `None` while more rounds remain to try, including when review never ran
5887/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
5888/// gone clean, or the budget is spent, or the tree has stopped moving (see
5889/// [`STAGNANT_LIMIT`]), the answer is one of two things:
5890///
5891/// - An incomplete panel that raised nothing is missing input, not a
5892///   verified tree — never a hand-off candidate, whatever verification said
5893///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
5894/// - Otherwise, green e2e on the last round hands off (see
5895///   [`Runner::stop_reviewing`]); red e2e blocks.
5896///
5897/// A last round whose own verification is still `ResourceBlocked` — magi
5898/// itself never got a command to run, not evidence the patch is broken —
5899/// is neither: this returns `None` for it too, the same as "more rounds
5900/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
5901/// handling of that shape) instead of this cheap recomputation guessing a
5902/// verdict a real attempt never produced.
5903fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
5904    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
5905        return Some(RunStatus::Gating);
5906    }
5907    let last = reviews.last()?;
5908    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
5909    if reviews.len() < max_rounds && !stagnant {
5910        return None;
5911    }
5912    if last.incomplete() && last.blocking == 0 {
5913        return Some(RunStatus::Blocked);
5914    }
5915    if last.e2e_status() == E2eStatus::ResourceBlocked {
5916        return None;
5917    }
5918    Some(if last.e2e.iter().all(CommandOutcome::ok) {
5919        RunStatus::Gating
5920    } else {
5921        RunStatus::Blocked
5922    })
5923}
5924
5925/// How long a re-ask may take, given the budget the first attempt had.
5926///
5927/// A `nudged` retry is a request to restate an answer the seat has already
5928/// worked out: it carries no new work, so it does not deserve the original
5929/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
5930/// 133 seconds while a third sat for over ten minutes on a resumed session
5931/// holding 410 KB of prior output - and because the retry had inherited the
5932/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
5933/// judging round whose other seats were long finished.
5934///
5935/// A quarter of the budget, with a floor so that a deliberately short timeout
5936/// does not collapse to nothing. A retry that re-sends the whole prompt
5937/// (because the seat kept no context) is the original job again, and keeps the
5938/// original budget.
5939fn retry_budget(full: Duration, nudged: bool) -> Duration {
5940    if nudged {
5941        (full / 4).max(Duration::from_secs(120)).min(full)
5942    } else {
5943        full
5944    }
5945}
5946
5947/// Run a wave and parse each reply, re-asking the seats whose reply was
5948/// unusable.
5949///
5950/// The re-ask is a nudge rather than the whole prompt again when the seat still
5951/// holds its conversation, which is the difference between a cheap retry and
5952/// paying for the entire candidate set twice.
5953///
5954/// A seat that hits a rate limit is **not** re-asked: the same call will fail
5955/// the same way until the limit resets, so spending a retry attempt on it is
5956/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
5957/// like any other absent seat — the caller decides whether the panel still has
5958/// a quorum.
5959#[allow(clippy::too_many_arguments)]
5960async fn ask_json_wave<T>(
5961    jobs: Vec<SeatJob>,
5962    sem: Arc<Semaphore>,
5963    retries: usize,
5964    ctx: &WaveCtx<'_>,
5965    losses: &mut Vec<QuotaLoss>,
5966    state: &mut RunState,
5967    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
5968) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
5969where
5970    T: serde::de::DeserializeOwned + Send + 'static,
5971{
5972    let n = jobs.len();
5973    let originals: Vec<SeatJob> = jobs;
5974    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
5975    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
5976    // Which attempt each seat's `done[i]` reflects — 0 for a first-ask
5977    // answer, N once it has gone through N nudges. Read back once this
5978    // returns, so a caller building a history record (`ReviewRecord`) can
5979    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
5980    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
5981    // field's own doc.
5982    let mut attempts_used: Vec<usize> = vec![0; n];
5983    let mut pending: Vec<usize> = (0..n).collect();
5984
5985    for attempt in 0..=retries {
5986        if pending.is_empty() {
5987            break;
5988        }
5989        let mut batch = Vec::with_capacity(pending.len());
5990        for &i in &pending {
5991            let src = &originals[i];
5992            // The prompt and the budget are one decision: a nudge restates
5993            // finished work, a re-sent prompt redoes it.
5994            let (prompt, timeout) = if attempt == 0 {
5995                (src.prompt.clone(), src.timeout)
5996            } else {
5997                let why = done[i]
5998                    .as_ref()
5999                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
6000                    .unwrap_or_else(|| "no parsable answer".to_owned());
6001                let nudge = prompt::nudge(&why);
6002                let nudged = has_context(&src.spec, &seats[i], src.sessions);
6003                let prompt = if nudged {
6004                    nudge
6005                } else {
6006                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
6007                };
6008                (prompt, retry_budget(src.timeout, nudged))
6009            };
6010            batch.push(SeatJob {
6011                spec: src.spec.clone(),
6012                seat: seats[i].clone(),
6013                cwd: src.cwd.clone(),
6014                prompt,
6015                timeout,
6016                allow_write: src.allow_write,
6017                sessions: src.sessions,
6018                artifacts: src.artifacts.clone(),
6019                stem: if attempt == 0 {
6020                    src.stem.clone()
6021                } else {
6022                    format!("{}-retry{attempt}", src.stem)
6023                },
6024            });
6025        }
6026
6027        if attempt > 0 {
6028            let seats_out: Vec<&str> = pending
6029                .iter()
6030                .map(|&i| originals[i].seat.key.as_str())
6031                .collect();
6032            state.event(
6033                ctx.node,
6034                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
6035            );
6036        }
6037        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
6038        let mut still = Vec::new();
6039        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
6040            seats[i] = seat;
6041            let (parsed, quota) = match out {
6042                AgentOutcome::Ok(o) => (
6043                    match verdict::extract_json::<T>(&o.text) {
6044                        Ok(v) => match validate(&v) {
6045                            Ok(()) => Ok((v, o)),
6046                            Err(e) => Err(e),
6047                        },
6048                        Err(e) => Err(e),
6049                    },
6050                    false,
6051                ),
6052                AgentOutcome::Quota(o) => {
6053                    losses.push(QuotaLoss {
6054                        seat: originals[i].seat.key.clone(),
6055                        node: ctx.node.to_owned(),
6056                        at: Timestamp::now(),
6057                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
6058                    });
6059                    (
6060                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
6061                        true,
6062                    )
6063                }
6064                // Not a parseable answer, but also not worth a special-cased
6065                // retry here: the nudge loop above already re-asks anything
6066                // that fails to parse, which is exactly what a dropped stream
6067                // needs. Just don't hand its raw error JSON to `extract_json`.
6068                AgentOutcome::Dropped(o) => {
6069                    let why = o
6070                        .dropped
6071                        .as_ref()
6072                        .map(|d| d.why.as_str())
6073                        .unwrap_or("the CLI ended the stream without delivering its answer");
6074                    (
6075                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
6076                        false,
6077                    )
6078                }
6079                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
6080            };
6081            let failed = parsed.is_err();
6082            done[i] = Some(parsed);
6083            attempts_used[i] = attempt;
6084            // Do not re-ask a rate-limited seat (quota) — a retry is known to
6085            // fail the same way; and never re-ask a seat that already parsed.
6086            if failed && !quota {
6087                still.push(i);
6088            }
6089        }
6090        pending = still;
6091    }
6092
6093    seats
6094        .into_iter()
6095        .zip(done)
6096        .zip(attempts_used)
6097        .map(|((seat, res), attempts)| {
6098            (
6099                seat,
6100                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
6101                attempts,
6102            )
6103        })
6104        .collect()
6105}
6106
6107/// Acquire the shared build cache's lease, waiting out contention within
6108/// `budget` (never past it — see AGENTS.md's build-cache section on why an
6109/// unbounded wait is never acceptable).
6110///
6111/// A first, non-blocking check happens before ever waiting; if it finds the
6112/// lease busy, that fact is logged as a `verify` event *and* flushed with
6113/// [`RunState::save`] immediately — not only once the wait finally succeeds
6114/// or gives up — so a `magi show` run by a different process while this one
6115/// is still waiting reads a `run.json` that says so, rather than whatever it
6116/// looked like before the wait started. The same applies to the terminal
6117/// failure: logged and saved before this returns `Err`, so a caller that
6118/// could not get the lease at all still leaves a legible record of why.
6119async fn acquire_cache_lease(
6120    state: &mut RunState,
6121    cache_dir: &Path,
6122    owner: &crate::cache::Owner,
6123    budget: Duration,
6124    context: &str,
6125) -> Result<crate::cache::Guard> {
6126    let home = crate::run::home();
6127    let started = Instant::now();
6128    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
6129        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
6130        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
6131        Err(e) => {
6132            state.event(
6133                "verify",
6134                format!("{context}: could not check the shared build cache: {e:#}"),
6135            );
6136            if let Err(e2) = state.save() {
6137                tracing::warn!("could not persist a cache-check failure: {e2:#}");
6138            }
6139            return Err(e);
6140        }
6141    };
6142    state.event(
6143        "verify",
6144        format!(
6145            "{context}: waiting for the shared build cache at {} ({})",
6146            cache_dir.display(),
6147            busy.describe()
6148        ),
6149    );
6150    if let Err(e) = state.save() {
6151        tracing::warn!("could not persist a cache wait: {e:#}");
6152    }
6153    let remaining = budget.saturating_sub(started.elapsed());
6154    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
6155        Ok(g) => Ok(g),
6156        Err(e) => {
6157            state.event("verify", format!("{context}: {e:#}"));
6158            if let Err(e2) = state.save() {
6159                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
6160            }
6161            Err(e)
6162        }
6163    }
6164}
6165
6166/// Run `body` — a verify command batch — while holding the shared build
6167/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
6168/// never interleave with another borrower's build against the same
6169/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
6170/// timeout, or a human's own `magi review`. See the `cache` module doc for
6171/// why this matters more than Cargo's own per-target locking covers — two
6172/// *different* worktrees building the same package name/version into one
6173/// cache directory is a staleness bug, not a lock contention one.
6174///
6175/// The wait for the lease is carved out of `budget`, never on top of it —
6176/// `body` is handed whatever is left, so a caller's own node timeout is the
6177/// only clock involved, exactly what AGENTS.md's build-cache section asks
6178/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
6179/// cache configured at all — this is a pass-through: `body` runs with the
6180/// full budget and nothing is leased.
6181///
6182/// A lease that cannot be acquired within `budget` is reported as a single
6183/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
6184/// verification — the same shape a spawn failure already takes in
6185/// [`run_commands`], so a caller need not special-case it.
6186#[allow(clippy::too_many_arguments)]
6187async fn with_cache_lease<'s, F, Fut>(
6188    state: &'s mut RunState,
6189    cache_dir: Option<&Path>,
6190    node: &str,
6191    seat: &str,
6192    worktree: &Path,
6193    head: &str,
6194    budget: Duration,
6195    context: &str,
6196    body: F,
6197) -> (Vec<CommandOutcome>, bool)
6198where
6199    F: FnOnce(&'s mut RunState, Duration) -> Fut,
6200    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
6201{
6202    let Some(cache_dir) = cache_dir else {
6203        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
6204        return (outcomes, retried);
6205    };
6206    let home = crate::run::home();
6207    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
6208    let started = Instant::now();
6209    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
6210        Ok(g) => g,
6211        Err(e) => {
6212            return (
6213                vec![CommandOutcome {
6214                    command: "(waiting for the shared build cache)".to_owned(),
6215                    code: None,
6216                    output_tail: e.to_string(),
6217                    duration_ms: started.elapsed().as_millis() as u64,
6218                    resource_blocked: true,
6219                }],
6220                false,
6221            );
6222        }
6223    };
6224    let identity = crate::cache::Identity::new(worktree, head);
6225    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
6226        // A failed freshness check means this process cannot vouch for what
6227        // is sitting in the cache right now - on Windows this is exactly the
6228        // "a stale test executable is still locked, `cargo clean -p` cannot
6229        // remove it" case the evidence log records. Running verify anyway
6230        // and reporting whatever it says would let a result nobody can trust
6231        // stand for the tree it claims to have checked; fail the step
6232        // instead of the patch.
6233        state.event(
6234            "verify",
6235            format!(
6236                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
6237                worktree.display(),
6238                short(head)
6239            ),
6240        );
6241        guard.release();
6242        return (
6243            vec![CommandOutcome {
6244                command: "(confirming the shared build cache is fresh)".to_owned(),
6245                code: None,
6246                output_tail: e.to_string(),
6247                duration_ms: started.elapsed().as_millis() as u64,
6248                resource_blocked: true,
6249            }],
6250            false,
6251        );
6252    }
6253    let remaining = budget.saturating_sub(started.elapsed());
6254    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
6255    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
6256    // `start_kill`); confirm it actually has before handing the directory to
6257    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
6258    // for what this can and cannot see.
6259    if !timed_out_pids.is_empty() {
6260        wait_for_timed_out_children_to_die(&timed_out_pids).await;
6261    }
6262    guard.release();
6263    (outcomes, retried)
6264}
6265
6266/// Poll `pids` — commands [`run_commands`] reports as still running when its
6267/// own timeout elapsed — until every one is confirmed gone, or
6268/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
6269///
6270/// Real confirmation where confirmation is possible, not a substitute for
6271/// full process-tree observation: a grandchild the timed-out process spawned
6272/// and that survives independently of it is invisible to a pid check the
6273/// same way it always was, and continuing to observe and collect *that*
6274/// stays a different piece of work with its own owner. This only narrows a
6275/// fixed blind wait into an actual check of the pids this process does know
6276/// about.
6277async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
6278    wait_for_pids_with(
6279        pids,
6280        crate::proc::pid_alive,
6281        LEASE_RELEASE_POLL,
6282        LEASE_RELEASE_MAX_WAIT,
6283    )
6284    .await;
6285}
6286
6287/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
6288/// interval and ceiling supplied by the caller, so the polling *logic* -
6289/// returns as soon as every pid reports dead, gives up at the ceiling
6290/// otherwise - is testable on millisecond durations without asking the real
6291/// OS about a pid at all.
6292async fn wait_for_pids_with<F: Fn(u32) -> bool>(
6293    pids: &[u32],
6294    alive: F,
6295    poll: Duration,
6296    max_wait: Duration,
6297) {
6298    let deadline = Instant::now() + max_wait;
6299    loop {
6300        if pids.iter().all(|&pid| !alive(pid)) {
6301            return;
6302        }
6303        if Instant::now() >= deadline {
6304            return;
6305        }
6306        tokio::time::sleep(poll).await;
6307    }
6308}
6309
6310/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
6311/// admission that it could not even get a verify command to run, as opposed
6312/// to evidence the command actually produced? A caller that would otherwise
6313/// read a resource-blocked outcome as a red command must check this first:
6314/// see [`Runner::gate`], which retries rather than records `Blocked` when
6315/// this is true.
6316fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
6317    outcomes.iter().any(|o| o.resource_blocked)
6318}
6319
6320/// What [`Runner::gate_fix_round`] decided.
6321enum GateFix {
6322    /// The tree changed and `verify.e2e` is still green: run the gate again.
6323    Retry,
6324    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
6325    /// last failure stands and the run ends blocked.
6326    Stop,
6327    /// `verify.e2e` could not run after the fix (magi's own contention):
6328    /// decide nothing now, a later reentry retries.
6329    Defer,
6330}
6331
6332/// Is every red command in `outcomes` an ordinary failure the code could
6333/// explain: it ran, exited non-zero, and said something?
6334///
6335/// A timeout, a spawn failure and a killed process all leave `code` `None`;
6336/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
6337/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
6338/// what the command is stays the gate's business.
6339fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
6340    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
6341    red.peek().is_some()
6342        && red.all(|o| {
6343            !o.resource_blocked
6344                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
6345                && !o.output_tail.trim().is_empty()
6346        })
6347}
6348
6349/// Describe one verify command's outcome for the event log, distinguishing a
6350/// build/link failure — the toolchain never produced a binary to run — from
6351/// an actual test failure, since only the latter is a verdict on the patch.
6352fn e2e_outcome_label(o: &CommandOutcome) -> String {
6353    if o.ok() {
6354        return "pass".to_owned();
6355    }
6356    let reason = if o.build_failed() {
6357        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
6358    } else {
6359        format!("FAIL ({:?})", o.code)
6360    };
6361    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
6362}
6363
6364/// Run `verify.e2e`, retrying once if the first attempt could not build or
6365/// link — a build/link failure is frequently a race against a shared
6366/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
6367/// `verify` event per command, tagged with `context` (normally `"round N"`)
6368/// so the two call sites that need this — the ordinary per-round leg in
6369/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
6370/// it will ever call a round green — read identically in the event log.
6371async fn run_e2e_with_retry(
6372    state: &mut RunState,
6373    shell: &[String],
6374    commands: &[String],
6375    worktree: &Path,
6376    timeout: Duration,
6377    context: &str,
6378) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
6379    let (mut e2e, mut timed_out_pids) = run_commands(
6380        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
6381    )
6382    .await;
6383    for o in &e2e {
6384        state.event(
6385            "verify",
6386            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
6387        );
6388    }
6389    // A build/link failure is not a verdict on the patch — it is frequently a
6390    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
6391    // one retry before letting a red like that decide the round.
6392    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
6393    if verify_retried {
6394        state.event(
6395            "verify",
6396            format!(
6397                "{context}: verify could not build/link, not a test result — retrying once \
6398                 before concluding"
6399            ),
6400        );
6401        let retried = run_commands(
6402            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
6403        )
6404        .await;
6405        e2e = retried.0;
6406        // Both attempts' timeouts matter, not just the last one: the first
6407        // attempt's descendants may still be alive alongside the retry's.
6408        timed_out_pids.extend(retried.1);
6409        for o in &e2e {
6410            state.event(
6411                "verify",
6412                format!(
6413                    "{context}: retry `{}` -> {}",
6414                    o.command,
6415                    e2e_outcome_label(o)
6416                ),
6417            );
6418        }
6419    }
6420    (e2e, verify_retried, timed_out_pids)
6421}
6422
6423/// Run configured shell commands in `cwd`, in order. The second element is
6424/// the pid of every command that hit `timeout` and was still running when
6425/// this stopped waiting on it (best-effort: `None` when the platform did not
6426/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
6427/// that releases a shared resource afterward needs to know.
6428///
6429/// Records `task` into [`RunState::active`] at every command boundary
6430/// (`RunState::task_command`) and clears it once the whole list has run
6431/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
6432/// for minutes with no seat and no output of its own to show for it (see
6433/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
6434/// apart from "not yet run" without this), and this is the only place that
6435/// knows which command is running right now and how many are left. Three
6436/// saves per command — start, not per second — matching the same "only at a
6437/// boundary" rule [`wave`] already follows for seats.
6438#[allow(clippy::too_many_arguments)]
6439async fn run_commands(
6440    state: &mut RunState,
6441    node: &str,
6442    task: &str,
6443    attempt: usize,
6444    shell: &[String],
6445    commands: &[String],
6446    cwd: &Path,
6447    timeout: Duration,
6448) -> (Vec<CommandOutcome>, Vec<u32>) {
6449    if commands.is_empty() {
6450        // Nothing to mark as running and nothing to clear — an empty list
6451        // means "not configured", and touching `active` (or the disk) over
6452        // that would be a write for every round of a repo with no
6453        // `verify.e2e` / `verify.gate` commands at all.
6454        return (Vec::new(), Vec::new());
6455    }
6456    let mut out = Vec::new();
6457    let mut timed_out_pids = Vec::new();
6458    let total = commands.len();
6459    for (idx, command) in commands.iter().enumerate() {
6460        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
6461        if let Err(e) = state.save() {
6462            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
6463        }
6464        let started = Instant::now();
6465        let mut cmd = tokio::process::Command::new(&shell[0]);
6466        cmd.quiet();
6467        cmd.args(&shell[1..])
6468            .arg(command)
6469            .current_dir(cwd)
6470            .stdin(std::process::Stdio::null())
6471            .stdout(std::process::Stdio::piped())
6472            .stderr(std::process::Stdio::piped())
6473            .kill_on_drop(true);
6474        let spawned = cmd.spawn();
6475        let (code, body) = match spawned {
6476            Ok(child) => {
6477                // Captured before the child is consumed below: `kill_on_drop`
6478                // only *asks* the process to die when the timeout branch
6479                // drops it, and the pid is the only way anyone downstream can
6480                // later check whether that request actually took.
6481                let pid = child.id();
6482                match tokio::time::timeout(timeout, child.wait_with_output()).await {
6483                    Ok(Ok(o)) => {
6484                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
6485                        body.push_str(&String::from_utf8_lossy(&o.stderr));
6486                        (o.status.code(), body)
6487                    }
6488                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
6489                    Err(_) => {
6490                        if let Some(pid) = pid {
6491                            timed_out_pids.push(pid);
6492                        }
6493                        (None, format!("timed out after {}s", timeout.as_secs()))
6494                    }
6495                }
6496            }
6497            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
6498        };
6499        out.push(CommandOutcome {
6500            command: command.clone(),
6501            code,
6502            output_tail: tail(&body, OUTPUT_TAIL),
6503            duration_ms: started.elapsed().as_millis() as u64,
6504            resource_blocked: false,
6505        });
6506    }
6507    state.task_finished(task);
6508    if let Err(e) = state.save() {
6509        tracing::warn!("could not persist the end of {task}: {e:#}");
6510    }
6511    (out, timed_out_pids)
6512}
6513
6514/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
6515/// section (`report::run`) and in the `merge` event this node records — for
6516/// the operator to run by hand.
6517///
6518/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
6519/// branch whose ruleset forbids merge commits (GitHub's "must not contain
6520/// merge commits", or "require linear history") rejects the push a `--no-ff`
6521/// merge would produce, which is exactly the guidance this function replaces.
6522/// `message`'s first line becomes the squash commit's subject, matching the
6523/// note `report::run` prints alongside this command — see that function for
6524/// why an explicit subject is not optional there.
6525fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
6526    let repo = repo.display();
6527    match style {
6528        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
6529        MergeStyle::Squash => {
6530            // The subject sits inside double quotes, and a title an agent
6531            // wrote may carry the characters that break out of them.
6532            let subject = message
6533                .lines()
6534                .next()
6535                .unwrap_or(branch)
6536                .replace(['\\', '"', '$', '`'], "");
6537            format!(
6538                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
6539            )
6540        }
6541        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
6542    }
6543}
6544
6545/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
6546/// under the hood, rejects a `title` over 256 characters and the whole
6547/// command fails — no PR at all, for a run whose body was otherwise fine
6548/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
6549/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
6550/// always how GitHub counts, plus one character for the trailing ellipsis
6551/// `title_from` may add. It is a margin, not a guarantee — a title packed
6552/// with multi-unit characters could still in principle land close to the
6553/// edge, but a real task title's occasional emoji or accented letter fits
6554/// comfortably inside it.
6555const PR_TITLE_MAX: usize = 240;
6556
6557/// What `merge = "pr"` (and the merge commit of the other modes) says about a
6558/// change: a title and a body describing what was *implemented*, not the task
6559/// that asked for it. A task reads as a request; a reader of the merged
6560/// history wants the change.
6561struct PrMessage {
6562    title: String,
6563    body: String,
6564}
6565
6566impl PrMessage {
6567    /// Title, blank line, body. The first line is the squash/merge commit
6568    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
6569    /// has to stay one sensible line.
6570    fn commit_message(&self) -> String {
6571        format!("{}\n\n{}", self.title, self.body)
6572    }
6573}
6574
6575/// The text after a leading `TITLE:` (any case) on `line`.
6576fn title_marker(line: &str) -> Option<&str> {
6577    let line = line.trim();
6578    let head = line.get(..6)?;
6579    head.eq_ignore_ascii_case("title:")
6580        .then(|| line[6..].trim())
6581}
6582
6583/// The implementer's own one-line title: the `TITLE:` line the implement
6584/// prompt asks for at the top of its SUMMARY. Candidate commits are all
6585/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
6586/// source, and a title that says as much is refused here too.
6587fn summary_title(summary: &str) -> Option<String> {
6588    let first = summary.lines().find(|l| !l.trim().is_empty())?;
6589    let raw = title_marker(first)?;
6590    if raw.is_empty() {
6591        return None;
6592    }
6593    let title = queue::title_from(raw, PR_TITLE_MAX);
6594    let lower = title.to_ascii_lowercase();
6595    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
6596        return None;
6597    }
6598    Some(title)
6599}
6600
6601/// `summary` without its `TITLE:` line, which the pull request title already
6602/// carries.
6603fn summary_without_title(summary: &str) -> String {
6604    let mut lines = summary.trim().lines().peekable();
6605    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
6606        lines.next();
6607    }
6608    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
6609}
6610
6611/// The pull request title and body for the winning candidate.
6612///
6613/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
6614/// the task's own opening line via [`queue::title_from`] when there is none.
6615/// `state.instruction` can open with blank lines (`task_text` only rejects a
6616/// body that is blank *entirely*), which `title_from` skips.
6617///
6618/// Body: the implementer's summary and the fixer's notes, then — when the
6619/// winning review round was not clean — the findings still open and whatever
6620/// the fixer declined, so `merge = "pr"` hands the reader the same material
6621/// `magi show` does. The task follows inside a collapsed block, and the
6622/// footer repeats the run and candidate as plain tags for a reader holding
6623/// only the merged commit or the PR body.
6624fn pr_message(state: &RunState, winner: char) -> PrMessage {
6625    let summary = state
6626        .candidates
6627        .iter()
6628        .find(|c| c.label == winner)
6629        .map(|c| c.summary.as_str())
6630        .unwrap_or_default();
6631    // The fallback is the operator's own words and may not be English; GitHub
6632    // text always is, so a non-English task gets a neutral title instead.
6633    let title = summary_title(summary).unwrap_or_else(|| {
6634        let t = queue::title_from(&state.instruction, PR_TITLE_MAX);
6635        if t.is_ascii() && t.chars().any(|c| c.is_ascii_alphabetic()) {
6636            t
6637        } else {
6638            format!(
6639                "chore: land candidate {} of run {}",
6640                winner.to_ascii_uppercase(),
6641                state.id
6642            )
6643        }
6644    });
6645
6646    let mut body = String::new();
6647    let what = summary_without_title(summary);
6648    if !what.is_empty() {
6649        body.push_str("## Summary\n\n");
6650        body.push_str(&what);
6651        body.push_str("\n\n");
6652    }
6653
6654    let fix = state.reviews.last().and_then(|r| r.fix.as_ref());
6655    if let Some(fix) = fix
6656        && !fix.notes.trim().is_empty()
6657    {
6658        body.push_str("## Review fixes\n\n");
6659        body.push_str(fix.notes.trim());
6660        body.push_str("\n\n");
6661    }
6662
6663    let open = state.open_findings();
6664    if !open.is_empty() {
6665        body.push_str("## Open review findings\n\n");
6666        for f in &open {
6667            body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
6668        }
6669        body.push('\n');
6670    }
6671
6672    if let Some(fix) = fix
6673        && !fix.rejected.is_empty()
6674    {
6675        body.push_str("## Declined by the fixer\n\n");
6676        for r in &fix.rejected {
6677            body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
6678        }
6679        body.push('\n');
6680    }
6681
6682    let task = state.instruction.trim();
6683    let task = if task.is_empty() {
6684        "(empty task)"
6685    } else {
6686        task
6687    };
6688    body.push_str(&format!(
6689        "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
6690        task.replace("</details>", "&lt;/details&gt;")
6691    ));
6692
6693    body.push_str(&format!(
6694        "\n---\nmagi:run/{} magi:candidate-{}\n",
6695        state.id,
6696        winner.to_ascii_lowercase()
6697    ));
6698
6699    // Prompts are advisory; this is the enforced half of the confidentiality
6700    // rule, and it covers the verbatim task in <details> too.
6701    let id = crate::scrub::Identity::current();
6702    PrMessage {
6703        title: crate::scrub::scrub(&title, &id),
6704        body: crate::scrub::scrub(&body, &id),
6705    }
6706}
6707
6708/// `gh pr create`, returning the PR url.
6709async fn gh_pr_create(
6710    cwd: &Path,
6711    base: &str,
6712    head: &str,
6713    title: &str,
6714    body: &str,
6715) -> Result<String> {
6716    let out = tokio::process::Command::new("gh")
6717        .args([
6718            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
6719        ])
6720        .current_dir(cwd)
6721        .quiet()
6722        .stdin(std::process::Stdio::null())
6723        .output()
6724        .await
6725        .context("spawn gh")?;
6726    if out.status.success() {
6727        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
6728    } else {
6729        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
6730    }
6731}
6732
6733/// Tear a run's worktrees and branches down.
6734///
6735/// `home` is where the updated `run.json` is saved (via
6736/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
6737/// a housekeeping pass already has its own honest `home` handed to it, and
6738/// falling through to the global here would write back through whichever
6739/// directory some other process or test pinned into that `OnceLock` first,
6740/// not the one the caller actually resolved its `runs` and `state` from.
6741pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
6742    let repo = state.repo.clone();
6743    let root = state.worktree_root();
6744    let winner = state.tally.as_ref().map(|t| t.winner);
6745    let mut removed = Vec::new();
6746
6747    for i in 0..state.candidates.len() {
6748        let c = state.candidates[i].clone();
6749        let is_winner = Some(c.label) == winner;
6750        if is_winner && !drop_winner {
6751            continue;
6752        }
6753        if c.worktree.exists() {
6754            git::worktree_remove(&repo, &c.worktree).await.ok();
6755            removed.push(c.worktree.to_string_lossy().into_owned());
6756        }
6757        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
6758            git::branch_delete(&repo, &c.branch).await.ok();
6759            removed.push(c.branch.clone());
6760        }
6761        state.candidates[i].folded = true;
6762    }
6763
6764    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
6765        let path = name.path();
6766        let keep = !drop_winner
6767            && winner.is_some_and(|w| {
6768                path.file_name()
6769                    .is_some_and(|n| n == format!("cand-{w}").as_str())
6770            });
6771        if keep {
6772            continue;
6773        }
6774        git::worktree_remove(&repo, &path).await.ok();
6775        removed.push(path.to_string_lossy().into_owned());
6776    }
6777
6778    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
6779    // judge worktrees, so once the loop above has cleared all of them out,
6780    // the parent is a bare directory nobody else was ever going to remove -
6781    // git only ever managed what was inside it. Left alone, one of these
6782    // accumulates per fully-folded run; the operator's own machine had 74.
6783    // `remove_if_empty` re-checks rather than assuming: a run whose winner
6784    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
6785    // so does anything a run never claimed that happens to share the bay.
6786    remove_if_empty(&root);
6787
6788    if state.enabled_worktree_config && drop_winner {
6789        // A release, not a raw disable: some sibling run in this repository
6790        // may still hold its own reference (see `git::acquire_worktree_config`),
6791        // and only the last release actually turns the setting back off.
6792        git::release_worktree_config(&repo).await.ok();
6793        state.enabled_worktree_config = false;
6794    }
6795    state.save_under(home)?;
6796    Ok(removed)
6797}
6798
6799/// Remove `dir` if it exists and has nothing in it.
6800///
6801/// Best-effort and silent by design: a directory that is not empty (a run
6802/// whose winner is still parked there, a stray file some other process left)
6803/// is exactly the case this must refuse, and a directory that is already gone
6804/// is not a failure worth reporting either. `std::fs::remove_dir` itself
6805/// already refuses a non-empty directory, so the emptiness check below is
6806/// belt, not suspenders - it is what keeps this from ever attempting the
6807/// removal in the case that matters, rather than trusting `remove_dir`'s
6808/// error path to have no side effects if it ever changed.
6809fn remove_if_empty(dir: &Path) {
6810    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
6811        std::fs::remove_dir(dir).ok();
6812    }
6813}
6814
6815/// Severity of the worst open finding in the last review round, for reporting.
6816pub fn worst_open(state: &RunState) -> Option<Severity> {
6817    state
6818        .reviews
6819        .last()?
6820        .reviews
6821        .iter()
6822        .flat_map(|r| r.findings.iter())
6823        .map(|f| f.severity)
6824        .max()
6825}
6826
6827#[cfg(test)]
6828mod tests {
6829    use super::*;
6830    use crate::run::GateStatus;
6831    use std::collections::BTreeMap;
6832    use std::time::Duration;
6833
6834    fn conductor() -> AgentSpec {
6835        AgentSpec {
6836            id: "conductor".to_owned(),
6837            kind: crate::config::AgentKind::Command,
6838            model: None,
6839            command: vec!["true".to_owned()],
6840            extra_args: Vec::new(),
6841            env: BTreeMap::new(),
6842            prompt_delivery: None,
6843        }
6844    }
6845
6846    fn spec(id: &str) -> AgentSpec {
6847        AgentSpec {
6848            id: id.to_owned(),
6849            kind: crate::config::AgentKind::Command,
6850            model: None,
6851            command: vec!["true".to_owned()],
6852            extra_args: Vec::new(),
6853            env: BTreeMap::new(),
6854            prompt_delivery: None,
6855        }
6856    }
6857
6858    // `next_untried_implementer` is the property `resume_quota_losses`'s own
6859    // fallback loop depends on to terminate: it must walk forward from the
6860    // seat's own position, never restart at the front of the roster, and it
6861    // must never hand back an id already tried, however many times that id
6862    // happens to appear.
6863
6864    #[test]
6865    fn next_untried_implementer_walks_forward_from_the_seats_own_position() {
6866        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6867        let tried = BTreeSet::from(["beta".to_owned()]);
6868        // beta sits at index 1; the next candidate is gamma, never alpha —
6869        // which is very likely a different candidate slot's own agent.
6870        let next = next_untried_implementer(&roster, 1, &tried);
6871        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
6872    }
6873
6874    #[test]
6875    fn next_untried_implementer_does_not_wrap_back_past_its_own_start() {
6876        let roster = vec![spec("alpha"), spec("beta")];
6877        let tried = BTreeSet::from(["beta".to_owned()]);
6878        // beta is the roster's last entry: nothing follows it, and alpha —
6879        // earlier in the roster, almost certainly a different candidate
6880        // slot's own agent — must not be reached by wrapping back to it.
6881        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6882    }
6883
6884    #[test]
6885    fn next_untried_implementer_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
6886        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
6887        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
6888        // beta (index 1) and gamma (index 2, the only entry after it) have
6889        // both been tried; alpha (index 0) never has, but it comes before
6890        // beta's own position, so there is nothing further for this seat.
6891        assert!(next_untried_implementer(&roster, 1, &tried).is_none());
6892    }
6893
6894    #[test]
6895    fn next_untried_implementer_skips_ids_already_tried_even_when_duplicated() {
6896        let roster = vec![spec("a"), spec("a"), spec("b")];
6897        let tried = BTreeSet::from(["a".to_owned()]);
6898        let next = next_untried_implementer(&roster, 0, &tried);
6899        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
6900    }
6901
6902    #[test]
6903    fn next_untried_implementer_returns_none_once_every_id_is_tried() {
6904        let roster = vec![spec("a"), spec("b")];
6905        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
6906        assert!(next_untried_implementer(&roster, 0, &tried).is_none());
6907    }
6908
6909    #[test]
6910    fn remove_if_empty_only_ever_takes_a_bare_directory() {
6911        let dir = tempfile::tempdir().unwrap();
6912        let bay = dir.path().join("ffff");
6913
6914        // Not there yet: nothing to do, nothing to panic on.
6915        remove_if_empty(&bay);
6916        assert!(!bay.exists());
6917
6918        // Something still inside - the winner's worktree, or a stray file -
6919        // keeps the directory standing.
6920        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
6921        remove_if_empty(&bay);
6922        assert!(bay.exists(), "non-empty directory must survive");
6923
6924        // Once the last entry is gone, so is the directory itself.
6925        std::fs::remove_dir(bay.join("cand-A")).unwrap();
6926        remove_if_empty(&bay);
6927        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
6928    }
6929
6930    // `round_is_clean` is the exact decision this task fixed: a round with a
6931    // seat that never answered must not read the same as a round every seat
6932    // actually reviewed. These are deterministic and process-free by design —
6933    // the equivalent end-to-end check (a real reviewer timing out under a
6934    // live graph run) is a genuine race against wall-clock contention, and a
6935    // spawn slow enough to blow even a generous budget under a loaded test
6936    // run must not turn this specific regression check flaky.
6937
6938    #[test]
6939    fn a_full_panel_that_found_nothing_is_clean() {
6940        assert!(round_is_clean(
6941            0,
6942            true,
6943            2,
6944            2,
6945            0,
6946            IncompleteReviewPolicy::Block
6947        ));
6948    }
6949
6950    #[test]
6951    fn a_missing_seat_is_never_clean_under_the_default_policy() {
6952        assert!(!round_is_clean(
6953            0,
6954            true,
6955            1,
6956            2,
6957            0,
6958            IncompleteReviewPolicy::Block
6959        ));
6960    }
6961
6962    #[test]
6963    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
6964        assert!(!round_is_clean(
6965            1,
6966            true,
6967            1,
6968            2,
6969            0,
6970            IncompleteReviewPolicy::Warn
6971        ));
6972    }
6973
6974    #[test]
6975    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
6976        assert!(round_is_clean(
6977            0,
6978            true,
6979            1,
6980            2,
6981            0,
6982            IncompleteReviewPolicy::Warn
6983        ));
6984    }
6985
6986    #[test]
6987    fn a_full_panel_with_an_open_finding_is_not_clean() {
6988        assert!(!round_is_clean(
6989            1,
6990            true,
6991            2,
6992            2,
6993            0,
6994            IncompleteReviewPolicy::Block
6995        ));
6996    }
6997
6998    #[test]
6999    fn a_full_panel_with_a_red_e2e_is_not_clean() {
7000        assert!(!round_is_clean(
7001            0,
7002            false,
7003            2,
7004            2,
7005            0,
7006            IncompleteReviewPolicy::Block
7007        ));
7008    }
7009
7010    // The stall this task closes: under the default `block` policy, a seat
7011    // missing only because it was rate limited must not force a wait for a
7012    // session limit that will not lift by the next round. `round_is_clean`
7013    // is where that quorum carve-out lives; the review loop around it never
7014    // changes what a reviewer's vote or a finding's severity means.
7015
7016    #[test]
7017    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
7018        // 1 of 2 answered, and the one missing was quota'd — the exact
7019        // "review-2 rate limited (quota)" shape from the field report.
7020        assert!(round_is_clean(
7021            0,
7022            true,
7023            1,
7024            2,
7025            1,
7026            IncompleteReviewPolicy::Block
7027        ));
7028    }
7029
7030    #[test]
7031    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
7032        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
7033        // not a quota loss (`quota_missing` stays 0) — worth another try.
7034        assert!(!round_is_clean(
7035            0,
7036            true,
7037            1,
7038            2,
7039            0,
7040            IncompleteReviewPolicy::Block
7041        ));
7042    }
7043
7044    #[test]
7045    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
7046        assert!(!round_is_clean(
7047            1,
7048            true,
7049            1,
7050            2,
7051            1,
7052            IncompleteReviewPolicy::Block
7053        ));
7054        assert!(!round_is_clean(
7055            0,
7056            false,
7057            1,
7058            2,
7059            1,
7060            IncompleteReviewPolicy::Block
7061        ));
7062    }
7063
7064    #[test]
7065    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
7066        // Every seat quota'd, nobody answered: there is no panel to decide
7067        // on, so this must fall through to the existing block-and-retry
7068        // fallback rather than call an unreviewed patch clean.
7069        assert!(!round_is_clean(
7070            0,
7071            true,
7072            0,
7073            2,
7074            2,
7075            IncompleteReviewPolicy::Block
7076        ));
7077    }
7078
7079    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
7080        CommandOutcome {
7081            command: "test".to_owned(),
7082            code,
7083            output_tail: String::new(),
7084            duration_ms: 0,
7085            resource_blocked,
7086        }
7087    }
7088
7089    #[test]
7090    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
7091        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
7092        assert!(
7093            !verify_inconclusive(&[outcome(Some(1), false)]),
7094            "an ordinary failure is still evidence about the patch"
7095        );
7096        assert!(verify_inconclusive(&[outcome(None, true)]));
7097        assert!(
7098            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
7099            "one inconclusive outcome taints the whole batch"
7100        );
7101        assert!(!verify_inconclusive(&[]));
7102    }
7103
7104    #[tokio::test]
7105    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
7106        // Alive for the first two checks, then dead - confirms the loop
7107        // actually re-polls rather than deciding once and sleeping out the
7108        // ceiling regardless.
7109        let calls = std::sync::atomic::AtomicUsize::new(0);
7110        let started = Instant::now();
7111        wait_for_pids_with(
7112            &[123],
7113            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
7114            Duration::from_millis(5),
7115            Duration::from_secs(5),
7116        )
7117        .await;
7118        assert!(
7119            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
7120            "must keep checking rather than deciding on the first answer"
7121        );
7122        assert!(
7123            started.elapsed() < Duration::from_secs(1),
7124            "must return the moment it is confirmed dead, not wait out the ceiling"
7125        );
7126    }
7127
7128    #[tokio::test]
7129    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
7130        let started = Instant::now();
7131        wait_for_pids_with(
7132            &[123],
7133            |_| true, // never reports dead
7134            Duration::from_millis(5),
7135            Duration::from_millis(30),
7136        )
7137        .await;
7138        let elapsed = started.elapsed();
7139        assert!(
7140            elapsed >= Duration::from_millis(30),
7141            "must not give up before its own ceiling: {elapsed:?}"
7142        );
7143        assert!(
7144            elapsed < Duration::from_secs(1),
7145            "must not wait past its own ceiling either: {elapsed:?}"
7146        );
7147    }
7148
7149    #[tokio::test]
7150    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
7151        let started = Instant::now();
7152        wait_for_pids_with(
7153            &[],
7154            |_| true,
7155            Duration::from_secs(5),
7156            Duration::from_secs(5),
7157        )
7158        .await;
7159        assert!(
7160            started.elapsed() < Duration::from_millis(200),
7161            "an empty pid list has nothing to confirm"
7162        );
7163    }
7164
7165    // `review_conclusion` is the exact decision the review hand-off task
7166    // fixed: a round budget spent (or a tree that stopped moving) must not
7167    // collapse into `Blocked` regardless of what verification actually
7168    // said. Deterministic and process-free for the same reason the
7169    // `round_is_clean` family above is.
7170    fn review_round(
7171        clean: bool,
7172        blocking: usize,
7173        answered: usize,
7174        expected: usize,
7175        progressed: bool,
7176        e2e_ok: bool,
7177    ) -> ReviewRound {
7178        ReviewRound {
7179            round: 1,
7180            head: "h".to_owned(),
7181            verified_head: None,
7182            verified_at: None,
7183            reviews: Vec::new(),
7184            e2e: vec![CommandOutcome {
7185                command: "test".to_owned(),
7186                code: Some(if e2e_ok { 0 } else { 1 }),
7187                output_tail: String::new(),
7188                duration_ms: 0,
7189                resource_blocked: false,
7190            }],
7191            verify_retried: false,
7192            e2e_deferred: false,
7193            e2e_defer_reason: None,
7194            fix: None,
7195            blocking,
7196            answered,
7197            expected,
7198            clean,
7199            progressed,
7200            vote_split: false,
7201            reconsideration: Vec::new(),
7202            verdict: None,
7203        }
7204    }
7205
7206    #[test]
7207    fn review_conclusion_is_none_when_nothing_has_run() {
7208        assert_eq!(review_conclusion(&[], 3), None);
7209    }
7210
7211    #[test]
7212    fn review_conclusion_is_none_while_rounds_remain() {
7213        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
7214        assert_eq!(review_conclusion(&rounds, 3), None);
7215    }
7216
7217    #[test]
7218    fn review_conclusion_is_gating_once_a_round_is_clean() {
7219        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
7220        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
7221    }
7222
7223    #[test]
7224    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
7225        let rounds = vec![
7226            review_round(false, 1, 2, 2, true, true),
7227            review_round(false, 1, 2, 2, true, true),
7228        ];
7229        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
7230    }
7231
7232    #[test]
7233    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
7234        let rounds = vec![
7235            review_round(false, 1, 2, 2, true, true),
7236            review_round(false, 1, 2, 2, true, false),
7237        ];
7238        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
7239    }
7240
7241    #[test]
7242    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
7243        // Magi never got a command to run against this round's own head — a
7244        // resource-blocked attempt, not a red one — so this must never
7245        // settle on `Blocked` the way a genuine e2e failure would. `None`
7246        // here is what tells `Runner::review_loop` to retry the check
7247        // itself rather than trust this cheap recomputation with a verdict
7248        // it cannot actually produce.
7249        let mut blocked = review_round(false, 1, 2, 2, true, false);
7250        blocked.e2e[0].resource_blocked = true;
7251        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
7252        assert_eq!(review_conclusion(&rounds, 2), None);
7253    }
7254
7255    #[test]
7256    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
7257        // Missing input, not a verified tree — never a hand-off candidate.
7258        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
7259        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
7260    }
7261
7262    #[test]
7263    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
7264        let rounds = vec![
7265            review_round(false, 1, 2, 2, false, true),
7266            review_round(false, 1, 2, 2, false, true),
7267        ];
7268        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
7269    }
7270
7271    fn secs(n: u64) -> Duration {
7272        Duration::from_secs(n)
7273    }
7274
7275    /// A throwaway repo with one commit on `main`, for tests that need `merge`
7276    /// to make real (and, if it runs at all, real*ly fail*) git calls.
7277    fn init_repo(dir: &Path) {
7278        let run = |args: &[&str]| {
7279            let out = std::process::Command::new("git")
7280                .args(args)
7281                .current_dir(dir)
7282                .quiet()
7283                .output()
7284                .expect("spawn git");
7285            assert!(
7286                out.status.success(),
7287                "git {args:?} failed: {}",
7288                String::from_utf8_lossy(&out.stderr)
7289            );
7290        };
7291        run(&["init", "-b", "main"]);
7292        run(&["config", "user.name", "magi test"]);
7293        run(&["config", "user.email", "magi@example.com"]);
7294        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
7295        run(&["add", "-A"]);
7296        run(&["commit", "-m", "init"]);
7297    }
7298
7299    // `settle_questions` is what closes the ghost the phone showed: a run's
7300    // seat asked something, the run then ended, and nothing was left to
7301    // abandon the question it left `open`. `HOME` is a process-wide
7302    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
7303    // first time it runs in the binary — every test below still reaches the
7304    // same directory whichever call won, and each gets its own run id from
7305    // `RunState::new`, so they never collide there.
7306    fn ask_test_home() {
7307        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
7308    }
7309
7310    /// A minimal, git-free `Runner` at a given status — `settle_questions`
7311    /// reads nothing else off it.
7312    fn runner_at(status: RunStatus) -> Runner {
7313        let mut state = RunState::new(
7314            PathBuf::from("/nonexistent/repo"),
7315            "main".to_owned(),
7316            "deadbeef".to_owned(),
7317            "task".to_owned(),
7318            Config::default(),
7319        );
7320        state.status = status;
7321        Runner {
7322            state,
7323            roles: ResolvedRoles {
7324                implementers: Vec::new(),
7325                judges: Vec::new(),
7326                reviewers: Vec::new(),
7327                fixer: None,
7328                conductor: conductor(),
7329                implementer_roster: Vec::new(),
7330            },
7331            sem: Arc::new(Semaphore::new(1)),
7332            pause: Pause::new(),
7333            interrupt: Pause::new(),
7334        }
7335    }
7336
7337    /// `park_here` folding in the reason `Pause::park_because` recorded -
7338    /// this is what lets an operator reading a run's events tell an
7339    /// interrupt-driven park from an ordinary shutdown park.
7340    #[test]
7341    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
7342        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7343        let mut runner = runner_at(RunStatus::Implementing);
7344        let interrupt = Pause::new();
7345        runner.watch_interrupt(interrupt.clone());
7346
7347        interrupt.park_because("task a1b2 asked to run first");
7348
7349        assert!(runner.park_here().expect("park_here"));
7350        assert!(runner.state.parked);
7351        let last = runner.state.events.last().expect("a park event");
7352        assert_eq!(last.node, "park");
7353        assert!(
7354            last.message.contains("task a1b2 asked to run first"),
7355            "expected the interrupt reason in {:?}",
7356            last.message
7357        );
7358    }
7359
7360    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
7361    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
7362    /// cleared) must not make a *different* run - one only watching its own,
7363    /// unshared interrupt `Pause` - see itself as parked. If a future change
7364    /// ever collapsed these back into one handle, the interrupt scheduler
7365    /// would park every run for the rest of the daemon's life, not just the
7366    /// one it meant to interrupt.
7367    #[test]
7368    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
7369        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7370        let mut runner = runner_at(RunStatus::Implementing);
7371        let shutdown = Pause::new();
7372        runner.on_pause(shutdown.clone());
7373        let interrupt = Pause::new();
7374        runner.watch_interrupt(interrupt.clone());
7375
7376        // Nobody has asked for anything yet.
7377        assert!(!runner.park_here().expect("park_here"));
7378        assert!(!runner.state.parked);
7379
7380        // Only the interrupt handle fires; the shutdown handle stays clear.
7381        interrupt.park_because("test");
7382        assert!(!shutdown.parked());
7383        assert!(runner.park_here().expect("park_here"));
7384    }
7385
7386    /// The property every prior attempt at this feature failed to pin down:
7387    /// asking a run to park while one of its nodes has a real, in-flight
7388    /// async operation running (an agent call, in production) must not cut
7389    /// that operation short. `park_here` is only ever consulted *between*
7390    /// `execute`'s node calls - see its own doc - so nothing inside a node
7391    /// can observe a park request until the node itself returns. This proves
7392    /// that structurally, with real `tokio` concurrency and a channel
7393    /// handshake (never a sleep, which would only prove "usually", not
7394    /// "cannot"): the "node" below reports that it has genuinely started,
7395    /// and only then is the park requested; the node still has to be told to
7396    /// finish before `park_here` is ever called, exactly mirroring every
7397    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
7398    /// in `execute`.
7399    #[tokio::test]
7400    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
7401        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7402        let mut runner = runner_at(RunStatus::Implementing);
7403        let interrupt = Pause::new();
7404        runner.watch_interrupt(interrupt.clone());
7405
7406        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
7407        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
7408
7409        // Stands in for one node's in-flight agent call: it proves it has
7410        // genuinely started, then blocks - exactly as a spawned CLI process
7411        // does - until told to finish.
7412        let node = async move {
7413            started_tx.send(()).expect("send started");
7414            finish_rx.await.expect("recv finish");
7415            "node finished"
7416        };
7417
7418        let interrupter = async move {
7419            started_rx.await.expect("recv started");
7420            // The call is now genuinely in flight. Ask it to park.
7421            interrupt.park_because("higher-priority task waiting");
7422            // Nothing the node does can observe this yet - there is no
7423            // check inside it, by construction - so let the executor run
7424            // anything pending and then let the node finish on its own.
7425            tokio::task::yield_now().await;
7426            finish_tx.send(()).expect("send finish");
7427        };
7428
7429        let (node_result, ()) = tokio::join!(node, interrupter);
7430        assert_eq!(
7431            node_result, "node finished",
7432            "the in-flight call ran to completion"
7433        );
7434
7435        // Only now, at the boundary the real `execute` would check right
7436        // after this node, does the park take effect.
7437        assert!(runner.park_here().expect("park_here"));
7438        assert!(runner.state.parked);
7439    }
7440
7441    /// A run parked mid-competition carries every field it had accumulated
7442    /// through the exact same disk round-trip an ordinary resume uses -
7443    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
7444    /// Nothing about parking for an interrupt is a special case of that path;
7445    /// this is what proves it rather than assuming it.
7446    #[test]
7447    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
7448        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
7449        let mut runner = runner_at(RunStatus::Judging);
7450        // `Runner::resume` re-resolves roles from the saved config, which
7451        // refuses an empty roster - give it the same minimal one `conductor`
7452        // itself uses.
7453        runner.state.config.agents = vec![conductor()];
7454        runner.state.candidates = vec![Candidate {
7455            index: 0,
7456            label: 'A',
7457            agent: "alpha".to_owned(),
7458            branch: "magi/x/A".to_owned(),
7459            worktree: PathBuf::from("/nonexistent/worktree"),
7460            summary: "did the thing".to_owned(),
7461            stat: "1 file changed".to_owned(),
7462            files: 1,
7463            commits: 1,
7464            empty: false,
7465            failed: None,
7466            verified_noop: None,
7467            duration_ms: 1234,
7468            folded: false,
7469        }];
7470        let run_id = runner.state.id.clone();
7471
7472        let interrupt = Pause::new();
7473        runner.watch_interrupt(interrupt.clone());
7474        interrupt.park_because("task c3d4 asked to run first");
7475        assert!(runner.park_here().expect("park_here"));
7476
7477        let resumed = Runner::resume(&run_id).expect("resume");
7478        assert_eq!(resumed.state.candidates.len(), 1);
7479        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
7480        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
7481        assert_eq!(resumed.state.status, runner.state.status);
7482        assert!(
7483            resumed.state.parked,
7484            "still parked until `execute` actually walks the graph again"
7485        );
7486        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
7487    }
7488
7489    /// A fresh open question on `run`, stored and handed back for assertions.
7490    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
7491        let mut q = ask::Question::new(
7492            run.to_owned(),
7493            "implement".to_owned(),
7494            "impl-A".to_owned(),
7495            "Which storage backend should the cache use?".to_owned(),
7496            String::new(),
7497            vec!["SQLite".to_owned(), "Redis".to_owned()],
7498        );
7499        store.put(&mut q).unwrap();
7500        q
7501    }
7502
7503    #[test]
7504    fn a_failed_runs_open_question_is_abandoned() {
7505        ask_test_home();
7506        let store = ask::Questions::open();
7507        let mut runner = runner_at(RunStatus::Failed);
7508        let run = runner.state.id.clone();
7509        let q = ask_open_question(&store, &run);
7510
7511        runner.settle_questions();
7512
7513        let back = store.get(&q.id).unwrap();
7514        assert!(
7515            !back.status.open(),
7516            "the seat that asked died with the run; nobody is left to read an answer"
7517        );
7518        assert!(
7519            back.detail.contains(&run) && back.detail.contains("failed"),
7520            "the reason names what the run became, not just that it is gone: {}",
7521            back.detail
7522        );
7523    }
7524
7525    #[test]
7526    fn a_merged_runs_open_question_is_abandoned_too() {
7527        ask_test_home();
7528        let store = ask::Questions::open();
7529        // A run that finishes cleanly still leaves nobody to read an answer -
7530        // this is not only a failure-path cleanup.
7531        for status in [RunStatus::Merged, RunStatus::Ready] {
7532            let mut runner = runner_at(status);
7533            let run = runner.state.id.clone();
7534            let q = ask_open_question(&store, &run);
7535
7536            runner.settle_questions();
7537
7538            let back = store.get(&q.id).unwrap();
7539            assert!(
7540                !back.status.open(),
7541                "{status:?} run's question must not outlive the run"
7542            );
7543        }
7544    }
7545
7546    #[test]
7547    fn a_still_resumable_runs_open_question_is_left_alone() {
7548        ask_test_home();
7549        let store = ask::Questions::open();
7550        // `Blocked` and `Stalled` can still be resumed — the candidates, the
7551        // review round and the seat sessions are all still on disk — so a
7552        // question asked mid-round may yet get a real answer from a real
7553        // resume. Sweeping it here would be exactly the failure mode this
7554        // whole feature exists to avoid on the other side.
7555        for status in [RunStatus::Blocked, RunStatus::Stalled] {
7556            let mut runner = runner_at(status);
7557            let run = runner.state.id.clone();
7558            let q = ask_open_question(&store, &run);
7559
7560            runner.settle_questions();
7561
7562            let back = store.get(&q.id).unwrap();
7563            assert!(
7564                back.status.open(),
7565                "{status:?} is still alive; the question must still be waiting"
7566            );
7567        }
7568    }
7569
7570    #[test]
7571    fn settle_questions_never_touches_an_already_answered_question() {
7572        ask_test_home();
7573        let store = ask::Questions::open();
7574        let mut runner = runner_at(RunStatus::Failed);
7575        let run = runner.state.id.clone();
7576        let mut q = ask_open_question(&store, &run);
7577        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
7578            .unwrap();
7579        store.put(&mut q).unwrap();
7580
7581        // Called twice, the way a crash-recovered daemon reclaim and the
7582        // graph's own cleanup both can for the same run — `abandon_for_run`
7583        // only ever touches what is still open, so this must be inert both
7584        // times, not merely the second.
7585        runner.settle_questions();
7586        runner.settle_questions();
7587
7588        let back = store.get(&q.id).unwrap();
7589        assert_eq!(
7590            back.status,
7591            ask::QuestionStatus::Answered,
7592            "a real answer is a decision on record, never overwritten by a sweep"
7593        );
7594    }
7595
7596    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
7597    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
7598    /// without merging, whose winner is still the operator's answer to read.
7599    /// Nothing previously called `fold_run` itself with a real `tally`, so
7600    /// this is the first test to pin down the one distinction the whole
7601    /// automatic-fold feature depends on: the winner's worktree and branch
7602    /// must survive, everything else sharing the run's worktree bay - a
7603    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
7604    /// second sweep treats every non-winner directory under the bay alike -
7605    /// must not.
7606    #[tokio::test]
7607    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
7608        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
7609        let tmp = tempfile::tempdir().expect("tempdir");
7610        let repo = tmp.path().join("repo");
7611        std::fs::create_dir_all(&repo).unwrap();
7612        init_repo(&repo);
7613
7614        let mut config = Config::default();
7615        config.graph.worktree_root = Some(tmp.path().join("wt"));
7616
7617        let mut state = RunState::new(
7618            repo.clone(),
7619            "main".to_owned(),
7620            "deadbeef".to_owned(),
7621            "task".to_owned(),
7622            config,
7623        );
7624        let root = state.worktree_root();
7625        let wt_a = root.join("cand-A");
7626        let wt_b = root.join("cand-B");
7627        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
7628            .await
7629            .expect("worktree A");
7630        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
7631            .await
7632            .expect("worktree B");
7633
7634        state.candidates = vec![
7635            Candidate {
7636                index: 0,
7637                label: 'A',
7638                agent: "alpha".to_owned(),
7639                branch: "magi/x/A".to_owned(),
7640                worktree: wt_a.clone(),
7641                summary: String::new(),
7642                stat: String::new(),
7643                files: 0,
7644                commits: 0,
7645                empty: false,
7646                failed: None,
7647                verified_noop: None,
7648                duration_ms: 0,
7649                folded: false,
7650            },
7651            Candidate {
7652                index: 1,
7653                label: 'B',
7654                agent: "beta".to_owned(),
7655                branch: "magi/x/B".to_owned(),
7656                worktree: wt_b.clone(),
7657                summary: String::new(),
7658                stat: String::new(),
7659                files: 0,
7660                commits: 0,
7661                empty: false,
7662                failed: None,
7663                verified_noop: None,
7664                duration_ms: 0,
7665                folded: false,
7666            },
7667        ];
7668        state.tally = Some(Tally {
7669            first_choice: BTreeMap::from([('A', 1)]),
7670            borda: BTreeMap::new(),
7671            winner: 'A',
7672            rankings: 1,
7673            unanimous_initial: true,
7674            deliberated: false,
7675            changed_votes: 0,
7676            unanimous_final: true,
7677            tie_break: None,
7678            judges: 1,
7679            present: 1,
7680            quorum: 1,
7681            met_quorum: true,
7682            uncontested: None,
7683        });
7684        state.status = RunStatus::Ready;
7685
7686        fold_run(&mut state, false, &crate::run::home())
7687            .await
7688            .expect("fold_run");
7689
7690        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
7691        assert!(
7692            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
7693            "the unmerged winner's branch survives"
7694        );
7695        assert!(
7696            !state.candidates[0].folded,
7697            "the winner is not marked folded"
7698        );
7699
7700        assert!(!wt_b.exists(), "the loser's worktree is removed");
7701        assert!(
7702            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
7703            "the loser's branch is removed"
7704        );
7705        assert!(state.candidates[1].folded, "the loser is marked folded");
7706    }
7707
7708    /// `status == Ready` used to be read as "this is the harmless
7709    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
7710    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
7711    /// run's PR was closed without merging — and reentering `merge` with
7712    /// `mode` still `Pr` does not know the difference, so it pushed and
7713    /// opened a second pull request. `mode == Local` reproduces the same
7714    /// blind spot without a network call: reentry must not attempt another
7715    /// git merge once this node has already recorded an outcome.
7716    #[tokio::test]
7717    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
7718        let tmp = tempfile::tempdir().expect("tempdir");
7719        let repo = tmp.path().join("repo");
7720        std::fs::create_dir_all(&repo).unwrap();
7721        init_repo(&repo);
7722
7723        let mut config = Config::default();
7724        config.merge.mode = MergeMode::Local;
7725
7726        let mut state = RunState::new(
7727            repo.clone(),
7728            "main".to_owned(),
7729            "deadbeef".to_owned(),
7730            "task".to_owned(),
7731            config,
7732        );
7733        state.candidates = vec![Candidate {
7734            index: 0,
7735            label: 'A',
7736            agent: "alpha".to_owned(),
7737            branch: "does-not-exist".to_owned(),
7738            worktree: repo.clone(),
7739            summary: String::new(),
7740            stat: String::new(),
7741            files: 0,
7742            commits: 0,
7743            empty: false,
7744            failed: None,
7745            verified_noop: None,
7746            duration_ms: 0,
7747            folded: false,
7748        }];
7749        state.tally = Some(Tally {
7750            first_choice: BTreeMap::from([('A', 1)]),
7751            borda: BTreeMap::new(),
7752            winner: 'A',
7753            rankings: 1,
7754            unanimous_initial: true,
7755            deliberated: false,
7756            changed_votes: 0,
7757            unanimous_final: true,
7758            tie_break: None,
7759            judges: 0,
7760            present: 0,
7761            quorum: 0,
7762            met_quorum: true,
7763            uncontested: Some("only candidate A produced a change".to_owned()),
7764        });
7765        state.reviews = vec![ReviewRound {
7766            round: 1,
7767            head: "deadbeef".to_owned(),
7768            verified_head: None,
7769            verified_at: None,
7770            reviews: Vec::new(),
7771            e2e: Vec::new(),
7772            fix: None,
7773            blocking: 0,
7774            answered: 0,
7775            expected: 0,
7776            clean: true,
7777            verify_retried: false,
7778            e2e_deferred: false,
7779            e2e_defer_reason: None,
7780            progressed: false,
7781            vote_split: false,
7782            reconsideration: Vec::new(),
7783            verdict: None,
7784        }];
7785        state.gate = vec![CommandOutcome {
7786            command: "test".to_owned(),
7787            code: Some(0),
7788            output_tail: String::new(),
7789            duration_ms: 0,
7790            resource_blocked: false,
7791        }];
7792        state.gate_ran = true;
7793        // Reached its conclusion already — e.g. `land` closing the PR without
7794        // merging it, which (like the honest `MergeMode::None` path) leaves
7795        // `status` at `Ready`. The recorded outcome is what actually marks
7796        // this node done.
7797        state.status = RunStatus::Ready;
7798        state.merge = Some(MergeOutcome {
7799            mode: MergeMode::Local,
7800            ok: false,
7801            detail: "already concluded".to_owned(),
7802        });
7803
7804        let mut runner = Runner {
7805            state,
7806            roles: ResolvedRoles {
7807                implementers: Vec::new(),
7808                judges: Vec::new(),
7809                reviewers: Vec::new(),
7810                fixer: None,
7811                conductor: conductor(),
7812                implementer_roster: Vec::new(),
7813            },
7814            sem: Arc::new(Semaphore::new(1)),
7815            pause: Pause::new(),
7816            interrupt: Pause::new(),
7817        };
7818
7819        runner.merge().await.expect("merge");
7820
7821        assert_eq!(
7822            runner.state.status,
7823            RunStatus::Ready,
7824            "a concluded run's status must not change on reentry"
7825        );
7826        assert_eq!(
7827            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
7828            Some("already concluded"),
7829            "merge must not run again once the node already recorded an outcome"
7830        );
7831    }
7832
7833    /// `gate` leaves `state.gate_ran` false both before it has ever run and
7834    /// when its last attempt was resource-blocked (the shared build cache
7835    /// could not be acquired or confirmed fresh in time - see
7836    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
7837    /// `Vec` this also leaves behind used to read as "nothing failed" and let
7838    /// a run merge a tree the gate never actually checked - exactly the case
7839    /// a contended cache produces on every retry until it clears. `merge`
7840    /// must refuse until `gate` has actually recorded an attempt.
7841    #[tokio::test]
7842    async fn merge_refuses_a_gate_that_has_not_actually_run() {
7843        let tmp = tempfile::tempdir().expect("tempdir");
7844        let repo = tmp.path().join("repo");
7845        std::fs::create_dir_all(&repo).unwrap();
7846        init_repo(&repo);
7847
7848        let mut config = Config::default();
7849        config.merge.mode = MergeMode::Local;
7850
7851        let mut state = RunState::new(
7852            repo.clone(),
7853            "main".to_owned(),
7854            "deadbeef".to_owned(),
7855            "task".to_owned(),
7856            config,
7857        );
7858        state.candidates = vec![Candidate {
7859            index: 0,
7860            label: 'A',
7861            agent: "alpha".to_owned(),
7862            branch: "does-not-exist".to_owned(),
7863            worktree: repo.clone(),
7864            summary: String::new(),
7865            stat: String::new(),
7866            files: 0,
7867            commits: 0,
7868            empty: false,
7869            failed: None,
7870            verified_noop: None,
7871            duration_ms: 0,
7872            folded: false,
7873        }];
7874        state.tally = Some(Tally {
7875            first_choice: BTreeMap::from([('A', 1)]),
7876            borda: BTreeMap::new(),
7877            winner: 'A',
7878            rankings: 1,
7879            unanimous_initial: true,
7880            deliberated: false,
7881            changed_votes: 0,
7882            unanimous_final: true,
7883            tie_break: None,
7884            judges: 0,
7885            present: 0,
7886            quorum: 0,
7887            met_quorum: true,
7888            uncontested: Some("only candidate A produced a change".to_owned()),
7889        });
7890        state.reviews = vec![ReviewRound {
7891            round: 1,
7892            head: "deadbeef".to_owned(),
7893            verified_head: None,
7894            verified_at: None,
7895            reviews: Vec::new(),
7896            e2e: Vec::new(),
7897            fix: None,
7898            blocking: 0,
7899            answered: 0,
7900            expected: 0,
7901            clean: true,
7902            verify_retried: false,
7903            e2e_deferred: false,
7904            e2e_defer_reason: None,
7905            progressed: false,
7906            vote_split: false,
7907            reconsideration: Vec::new(),
7908            verdict: None,
7909        }];
7910        // The point: `gate` has not recorded anything yet.
7911        state.gate = Vec::new();
7912        state.gate_ran = false;
7913        state.status = RunStatus::Gating;
7914
7915        let mut runner = Runner {
7916            state,
7917            roles: ResolvedRoles {
7918                implementers: Vec::new(),
7919                judges: Vec::new(),
7920                reviewers: Vec::new(),
7921                fixer: None,
7922                conductor: conductor(),
7923                implementer_roster: Vec::new(),
7924            },
7925            sem: Arc::new(Semaphore::new(1)),
7926            pause: Pause::new(),
7927            interrupt: Pause::new(),
7928        };
7929
7930        runner.merge().await.expect("merge");
7931
7932        assert!(
7933            runner.state.merge.is_none(),
7934            "an empty gate must never be read as a passing one: {:?}",
7935            runner.state.merge
7936        );
7937    }
7938
7939    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
7940    /// commands configured and `merge.mode` is `none` (a review-only run).
7941    /// `gate` must still record a real attempt — zero commands, vacuously
7942    /// passed — rather than leaving `state.gate` empty in a way `merge`
7943    /// cannot tell apart from "never ran"; otherwise the run reaches
7944    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
7945    #[tokio::test]
7946    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
7947        let tmp = tempfile::tempdir().expect("tempdir");
7948        let repo = tmp.path().join("repo");
7949        std::fs::create_dir_all(&repo).unwrap();
7950        init_repo(&repo);
7951
7952        // Default config: `verify.gate` empty, `merge.mode` is `none`.
7953        let config = Config::default();
7954
7955        let mut state = RunState::new(
7956            repo.clone(),
7957            "main".to_owned(),
7958            "deadbeef".to_owned(),
7959            "task".to_owned(),
7960            config,
7961        );
7962        state.candidates = vec![Candidate {
7963            index: 0,
7964            label: 'A',
7965            agent: "alpha".to_owned(),
7966            branch: "does-not-exist".to_owned(),
7967            worktree: repo.clone(),
7968            summary: String::new(),
7969            stat: String::new(),
7970            files: 0,
7971            commits: 0,
7972            empty: false,
7973            failed: None,
7974            verified_noop: None,
7975            duration_ms: 0,
7976            folded: false,
7977        }];
7978        state.tally = Some(Tally {
7979            first_choice: BTreeMap::from([('A', 1)]),
7980            borda: BTreeMap::new(),
7981            winner: 'A',
7982            rankings: 1,
7983            unanimous_initial: true,
7984            deliberated: false,
7985            changed_votes: 0,
7986            unanimous_final: true,
7987            tie_break: None,
7988            judges: 0,
7989            present: 0,
7990            quorum: 0,
7991            met_quorum: true,
7992            uncontested: Some("only candidate A produced a change".to_owned()),
7993        });
7994        state.reviews = vec![ReviewRound {
7995            round: 1,
7996            head: "deadbeef".to_owned(),
7997            verified_head: None,
7998            verified_at: None,
7999            reviews: Vec::new(),
8000            e2e: Vec::new(),
8001            fix: None,
8002            blocking: 0,
8003            answered: 0,
8004            expected: 0,
8005            clean: true,
8006            verify_retried: false,
8007            e2e_deferred: false,
8008            e2e_defer_reason: None,
8009            progressed: false,
8010            vote_split: false,
8011            reconsideration: Vec::new(),
8012            verdict: None,
8013        }];
8014
8015        let mut runner = Runner {
8016            state,
8017            roles: ResolvedRoles {
8018                implementers: Vec::new(),
8019                judges: Vec::new(),
8020                reviewers: Vec::new(),
8021                fixer: None,
8022                conductor: conductor(),
8023                implementer_roster: Vec::new(),
8024            },
8025            sem: Arc::new(Semaphore::new(1)),
8026            pause: Pause::new(),
8027            interrupt: Pause::new(),
8028        };
8029
8030        runner.gate().await.expect("gate");
8031        assert!(
8032            runner.state.gate_ran,
8033            "zero configured commands is still a real attempt, not an unrun gate"
8034        );
8035        assert!(runner.state.gate.is_empty());
8036        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
8037        assert_ne!(
8038            runner.state.status,
8039            RunStatus::Blocked,
8040            "a gate with nothing to check must not read as failed"
8041        );
8042
8043        runner.merge().await.expect("merge");
8044        assert_eq!(
8045            runner.state.status,
8046            RunStatus::Ready,
8047            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
8048        );
8049    }
8050
8051    /// `Config::cache_dir` is derived from `verify.e2e` as well as
8052    /// `verify.gate` (so the e2e leg and the final gate never build against
8053    /// different directories). With zero `verify.gate` commands but a
8054    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
8055    /// that lease before discovering it had nothing to run - so a repo with
8056    /// no gate commands could come back `resource_blocked` (and therefore
8057    /// still `gate_ran == false`) on nothing but an unrelated run holding the
8058    /// cache, exactly the contention this run's own zero commands could
8059    /// never have touched. `gate` must recognise there is nothing to check
8060    /// before it ever asks for the lease.
8061    #[tokio::test]
8062    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
8063        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8064        let home = crate::run::home();
8065
8066        let tmp = tempfile::tempdir().expect("tempdir");
8067        let repo = tmp.path().join("repo");
8068        std::fs::create_dir_all(&repo).unwrap();
8069        init_repo(&repo);
8070        // Unique to this test, so holding its lease cannot collide with
8071        // another test sharing the same process-wide `home`.
8072        let cache_dir = tmp.path().join("target");
8073
8074        let mut config = Config::default();
8075        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
8076        // `verify.gate` stays empty (the default). Bounded so a regression
8077        // that does start waiting fails the test in seconds, not hangs it.
8078        config.graph.timeout_verify = Some(2);
8079
8080        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8081        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8082            .expect("no io error acquiring directly")
8083        {
8084            crate::cache::AcquireOutcome::Acquired(g) => g,
8085            crate::cache::AcquireOutcome::Busy(b) => {
8086                panic!("expected the direct acquire to win the lease first: {b:?}")
8087            }
8088        };
8089
8090        let mut state = RunState::new(
8091            repo.clone(),
8092            "main".to_owned(),
8093            "deadbeef".to_owned(),
8094            "task".to_owned(),
8095            config,
8096        );
8097        state.candidates = vec![Candidate {
8098            index: 0,
8099            label: 'A',
8100            agent: "alpha".to_owned(),
8101            branch: "does-not-exist".to_owned(),
8102            worktree: repo.clone(),
8103            summary: String::new(),
8104            stat: String::new(),
8105            files: 0,
8106            commits: 0,
8107            empty: false,
8108            failed: None,
8109            verified_noop: None,
8110            duration_ms: 0,
8111            folded: false,
8112        }];
8113        state.tally = Some(Tally {
8114            first_choice: BTreeMap::from([('A', 1)]),
8115            borda: BTreeMap::new(),
8116            winner: 'A',
8117            rankings: 1,
8118            unanimous_initial: true,
8119            deliberated: false,
8120            changed_votes: 0,
8121            unanimous_final: true,
8122            tie_break: None,
8123            judges: 0,
8124            present: 0,
8125            quorum: 0,
8126            met_quorum: true,
8127            uncontested: Some("only candidate A produced a change".to_owned()),
8128        });
8129        state.reviews = vec![ReviewRound {
8130            round: 1,
8131            head: "deadbeef".to_owned(),
8132            verified_head: None,
8133            verified_at: None,
8134            reviews: Vec::new(),
8135            e2e: Vec::new(),
8136            fix: None,
8137            blocking: 0,
8138            answered: 0,
8139            expected: 0,
8140            clean: true,
8141            verify_retried: false,
8142            e2e_deferred: false,
8143            e2e_defer_reason: None,
8144            progressed: false,
8145            vote_split: false,
8146            reconsideration: Vec::new(),
8147            verdict: None,
8148        }];
8149
8150        let mut runner = Runner {
8151            state,
8152            roles: ResolvedRoles {
8153                implementers: Vec::new(),
8154                judges: Vec::new(),
8155                reviewers: Vec::new(),
8156                fixer: None,
8157                conductor: conductor(),
8158                implementer_roster: Vec::new(),
8159            },
8160            sem: Arc::new(Semaphore::new(1)),
8161            pause: Pause::new(),
8162            interrupt: Pause::new(),
8163        };
8164
8165        let started = std::time::Instant::now();
8166        runner.gate().await.expect("gate");
8167        assert!(
8168            started.elapsed() < Duration::from_secs(1),
8169            "a gate with nothing to run must never wait on a lease it never needed"
8170        );
8171        assert!(
8172            runner.state.gate_ran,
8173            "zero commands is still a real, immediate attempt"
8174        );
8175        assert!(runner.state.gate.is_empty());
8176        assert_ne!(
8177            runner.state.status,
8178            RunStatus::Blocked,
8179            "must not read as resource-blocked on a lease it never asked for"
8180        );
8181    }
8182
8183    /// The addendum's second gap: a `verify.gate` command running for real
8184    /// wall-clock time had nothing at all to show for it in `active` before
8185    /// `run_commands` learned to record it — a run could sit in `Gating` for
8186    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
8187    /// what was actually happening. Proven with a genuinely still-running
8188    /// command, not just a before/after check on the final state: a poller
8189    /// task reads the same `run.json` `gate()` is writing, the same way the
8190    /// phone or `magi show` would, while the shell command is still blocked
8191    /// on its own release marker.
8192    #[tokio::test]
8193    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
8194        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8195
8196        let tmp = tempfile::tempdir().expect("tempdir");
8197        let repo = tmp.path().join("repo");
8198        std::fs::create_dir_all(&repo).unwrap();
8199        init_repo(&repo);
8200
8201        let mut config = Config::default();
8202        config.verify.gate = vec![
8203            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
8204             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
8205                .to_owned(),
8206        ];
8207
8208        let mut state = RunState::new(
8209            repo.clone(),
8210            "main".to_owned(),
8211            "deadbeef".to_owned(),
8212            "task".to_owned(),
8213            config,
8214        );
8215        let run_id = state.id.clone();
8216        state.candidates = vec![Candidate {
8217            index: 0,
8218            label: 'A',
8219            agent: "alpha".to_owned(),
8220            branch: "does-not-exist".to_owned(),
8221            worktree: repo.clone(),
8222            summary: String::new(),
8223            stat: String::new(),
8224            files: 0,
8225            commits: 0,
8226            empty: false,
8227            failed: None,
8228            verified_noop: None,
8229            duration_ms: 0,
8230            folded: false,
8231        }];
8232        state.tally = Some(Tally {
8233            first_choice: BTreeMap::from([('A', 1)]),
8234            borda: BTreeMap::new(),
8235            winner: 'A',
8236            rankings: 1,
8237            unanimous_initial: true,
8238            deliberated: false,
8239            changed_votes: 0,
8240            unanimous_final: true,
8241            tie_break: None,
8242            judges: 0,
8243            present: 0,
8244            quorum: 0,
8245            met_quorum: true,
8246            uncontested: Some("only candidate A produced a change".to_owned()),
8247        });
8248        state.reviews = vec![ReviewRound {
8249            round: 1,
8250            head: "deadbeef".to_owned(),
8251            verified_head: None,
8252            verified_at: None,
8253            reviews: Vec::new(),
8254            e2e: Vec::new(),
8255            fix: None,
8256            blocking: 0,
8257            answered: 0,
8258            expected: 0,
8259            clean: true,
8260            verify_retried: false,
8261            e2e_deferred: false,
8262            e2e_defer_reason: None,
8263            progressed: false,
8264            vote_split: false,
8265            reconsideration: Vec::new(),
8266            verdict: None,
8267        }];
8268
8269        let mut runner = Runner {
8270            state,
8271            roles: ResolvedRoles {
8272                implementers: Vec::new(),
8273                judges: Vec::new(),
8274                reviewers: Vec::new(),
8275                fixer: None,
8276                conductor: conductor(),
8277                implementer_roster: Vec::new(),
8278            },
8279            sem: Arc::new(Semaphore::new(1)),
8280            pause: Pause::new(),
8281            interrupt: Pause::new(),
8282        };
8283
8284        let started_marker = repo.join("started.marker");
8285        let release_marker = repo.join("release.marker");
8286        let poller = tokio::spawn(async move {
8287            // Bounded so a regression that never records the task entry
8288            // fails this test in seconds instead of hanging the suite —
8289            // the same shape `a_park_requested_while_a_seat_is_mid_call_
8290            // does_not_cut_it_short` uses for the same reason.
8291            for _ in 0..100 {
8292                if started_marker.exists()
8293                    && let Ok(s) = crate::run::RunState::load(&run_id)
8294                    && let Some(a) = s.active.get("gate")
8295                {
8296                    std::fs::write(&release_marker, b"go").expect("release marker");
8297                    return Some(a.clone());
8298                }
8299                tokio::time::sleep(Duration::from_millis(50)).await;
8300            }
8301            None
8302        });
8303
8304        runner.gate().await.expect("gate");
8305        let captured = poller.await.expect("poller task");
8306        let captured = captured.expect(
8307            "the poller never saw a `gate` task entry in run.json while the command was \
8308             still blocked on its own release marker",
8309        );
8310
8311        assert_eq!(captured.task.as_deref(), Some("gate"));
8312        assert_eq!(captured.node, "gate");
8313        assert_eq!(captured.index, Some(1));
8314        assert_eq!(captured.total, Some(1));
8315        assert!(
8316            captured
8317                .command
8318                .as_deref()
8319                .is_some_and(|c| c.contains("started.marker")),
8320            "{captured:?}"
8321        );
8322
8323        assert!(
8324            runner.state.active.is_empty(),
8325            "the entry must be cleared once the command actually finished: {:?}",
8326            runner.state.active
8327        );
8328        assert!(runner.state.gate_ran);
8329        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
8330    }
8331
8332    /// The shape the incident this whole fix responds to actually had: the
8333    /// round budget spent, the last round's own e2e blocked on the shared
8334    /// build cache (held here by a live pid — this test process — exactly
8335    /// `cache`'s own unit tests' pattern for "another owner, still alive"
8336    /// without forking a process). `stop_reviewing` must retry it — not
8337    /// silently leave the round looking untouched (the catch-up-only half of
8338    /// the bug), and not read the contention as a red `e2e` and block the
8339    /// run on it (the other half). Called directly, the same way
8340    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
8341    /// above exercises `gate`, so this never needs a real cargo build to
8342    /// reach: the lease is never released, so `with_cache_lease` never gets
8343    /// past acquiring it into anything that would need a real workspace.
8344    #[tokio::test]
8345    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
8346        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8347        let home = crate::run::home();
8348
8349        let tmp = tempfile::tempdir().expect("tempdir");
8350        let repo = tmp.path().join("repo");
8351        std::fs::create_dir_all(&repo).unwrap();
8352        init_repo(&repo);
8353        let head = crate::git::rev_parse(&repo, "HEAD")
8354            .await
8355            .expect("rev-parse");
8356        // Unique to this test, so holding its lease cannot collide with
8357        // another test sharing the same process-wide `home`.
8358        let cache_dir = tmp.path().join("target");
8359
8360        let mut config = Config::default();
8361        config.verify.e2e = vec![format!(
8362            "CARGO_TARGET_DIR='{}' test -f README.md",
8363            cache_dir.display()
8364        )];
8365        config.graph.review_rounds = 1;
8366        // Bounded so a regression that does start waiting fails the test in
8367        // seconds, not hangs it.
8368        config.graph.timeout_verify = Some(2);
8369
8370        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8371        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8372            .expect("no io error acquiring directly")
8373        {
8374            crate::cache::AcquireOutcome::Acquired(g) => g,
8375            crate::cache::AcquireOutcome::Busy(b) => {
8376                panic!("expected the direct acquire to win the lease first: {b:?}")
8377            }
8378        };
8379
8380        let mut state = RunState::new(
8381            repo.clone(),
8382            "main".to_owned(),
8383            head.clone(),
8384            "task".to_owned(),
8385            config,
8386        );
8387        state.candidates = vec![Candidate {
8388            index: 0,
8389            label: 'A',
8390            agent: "alpha".to_owned(),
8391            branch: "does-not-exist".to_owned(),
8392            worktree: repo.clone(),
8393            summary: String::new(),
8394            stat: String::new(),
8395            files: 0,
8396            commits: 0,
8397            empty: false,
8398            failed: None,
8399            verified_noop: None,
8400            duration_ms: 0,
8401            folded: false,
8402        }];
8403        state.tally = Some(Tally {
8404            first_choice: BTreeMap::from([('A', 1)]),
8405            borda: BTreeMap::new(),
8406            winner: 'A',
8407            rankings: 1,
8408            unanimous_initial: true,
8409            deliberated: false,
8410            changed_votes: 0,
8411            unanimous_final: true,
8412            tie_break: None,
8413            judges: 0,
8414            present: 0,
8415            quorum: 0,
8416            met_quorum: true,
8417            uncontested: Some("only candidate A produced a change".to_owned()),
8418        });
8419        // The round budget's last round, deferred: `needs_catchup_run`'s
8420        // other trigger. `stop_reviewing`'s retry machinery must treat this
8421        // exactly like a resource-blocked attempt once it actually runs.
8422        state.reviews = vec![ReviewRound {
8423            round: 1,
8424            head: head.clone(),
8425            verified_head: None,
8426            verified_at: None,
8427            reviews: Vec::new(),
8428            e2e: Vec::new(),
8429            fix: None,
8430            blocking: 1,
8431            answered: 1,
8432            expected: 1,
8433            clean: false,
8434            verify_retried: false,
8435            e2e_deferred: true,
8436            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
8437            progressed: false,
8438            vote_split: false,
8439            reconsideration: Vec::new(),
8440            verdict: None,
8441        }];
8442
8443        let mut runner = Runner {
8444            state,
8445            roles: ResolvedRoles {
8446                implementers: Vec::new(),
8447                judges: Vec::new(),
8448                reviewers: Vec::new(),
8449                fixer: None,
8450                conductor: conductor(),
8451                implementer_roster: Vec::new(),
8452            },
8453            sem: Arc::new(Semaphore::new(1)),
8454            pause: Pause::new(),
8455            interrupt: Pause::new(),
8456        };
8457
8458        let shell = runner.state.config.shell();
8459        runner
8460            .stop_reviewing("round budget spent", &shell, &repo)
8461            .await
8462            .expect("stop_reviewing");
8463
8464        let last = runner.state.reviews.last().expect("round record");
8465        assert_eq!(
8466            last.e2e_status(),
8467            E2eStatus::ResourceBlocked,
8468            "the shared cache is still held; the attempt must read as blocked, not deferred or \
8469             failed: {last:?}"
8470        );
8471        assert_eq!(
8472            last.verified_head.as_deref(),
8473            Some(head.as_str()),
8474            "which commit this attempt targeted is known even though nothing finished checking \
8475             it"
8476        );
8477        let first_attempt_at = last
8478            .verified_at
8479            .expect("when this attempt ran is known too");
8480        assert_ne!(
8481            runner.state.status,
8482            RunStatus::Blocked,
8483            "contention is evidence about the machine, not the patch — it must not settle the \
8484             run as blocked: {:?}",
8485            runner.state.status
8486        );
8487        assert!(
8488            !runner
8489                .state
8490                .events
8491                .iter()
8492                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
8493            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
8494            runner.state.events
8495        );
8496
8497        // The cache is still held: a later reentry must retry the same
8498        // round's verification again — not leave it looking exactly as
8499        // untouched as the first blocked attempt, which is indistinguishable
8500        // from never having tried again at all.
8501        runner
8502            .stop_reviewing("round budget spent", &shell, &repo)
8503            .await
8504            .expect("stop_reviewing retry");
8505        assert_eq!(
8506            runner.state.reviews.len(),
8507            1,
8508            "no new round was started: {:?}",
8509            runner.state.reviews
8510        );
8511        let last = runner.state.reviews.last().expect("round record");
8512        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
8513        assert!(
8514            last.verified_at.expect("still known") > first_attempt_at,
8515            "a second reentry must be a fresh attempt, not a stale copy of the first"
8516        );
8517        assert_ne!(runner.state.status, RunStatus::Blocked);
8518
8519        held.release();
8520    }
8521
8522    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
8523    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
8524    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
8525    /// own top-of-function fast path (`review_conclusion`) correctly reads
8526    /// this shape as `None` rather than guessing `Blocked`, and the loop's
8527    /// own `for` range is empty once the round budget is spent, so
8528    /// `review_loop` must retry the check itself rather than silently doing
8529    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
8530    /// above exercises directly, but through `review_loop`'s own entry point
8531    /// this time, proving the wiring between the two rather than just the
8532    /// retry logic in isolation.
8533    #[tokio::test]
8534    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
8535        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8536        let home = crate::run::home();
8537
8538        let tmp = tempfile::tempdir().expect("tempdir");
8539        let repo = tmp.path().join("repo");
8540        std::fs::create_dir_all(&repo).unwrap();
8541        init_repo(&repo);
8542        let head = crate::git::rev_parse(&repo, "HEAD")
8543            .await
8544            .expect("rev-parse");
8545        let cache_dir = tmp.path().join("target");
8546
8547        let mut config = Config::default();
8548        config.verify.e2e = vec![format!(
8549            "CARGO_TARGET_DIR='{}' test -f README.md",
8550            cache_dir.display()
8551        )];
8552        config.graph.review_rounds = 1;
8553        config.graph.timeout_verify = Some(2);
8554
8555        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
8556        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
8557            .expect("no io error acquiring directly")
8558        {
8559            crate::cache::AcquireOutcome::Acquired(g) => g,
8560            crate::cache::AcquireOutcome::Busy(b) => {
8561                panic!("expected the direct acquire to win the lease first: {b:?}")
8562            }
8563        };
8564
8565        let mut state = RunState::new(
8566            repo.clone(),
8567            "main".to_owned(),
8568            head.clone(),
8569            "task".to_owned(),
8570            config,
8571        );
8572        state.candidates = vec![Candidate {
8573            index: 0,
8574            label: 'A',
8575            agent: "alpha".to_owned(),
8576            branch: "does-not-exist".to_owned(),
8577            worktree: repo.clone(),
8578            summary: String::new(),
8579            stat: String::new(),
8580            files: 0,
8581            commits: 0,
8582            empty: false,
8583            failed: None,
8584            verified_noop: None,
8585            duration_ms: 0,
8586            folded: false,
8587        }];
8588        state.tally = Some(Tally {
8589            first_choice: BTreeMap::from([('A', 1)]),
8590            borda: BTreeMap::new(),
8591            winner: 'A',
8592            rankings: 1,
8593            unanimous_initial: true,
8594            deliberated: false,
8595            changed_votes: 0,
8596            unanimous_final: true,
8597            tie_break: None,
8598            judges: 0,
8599            present: 0,
8600            quorum: 0,
8601            met_quorum: true,
8602            uncontested: Some("only candidate A produced a change".to_owned()),
8603        });
8604        // The exact shape a prior process's `stop_reviewing` would have left
8605        // on disk: the round budget's last round, a real attempt already
8606        // made and already resource-blocked.
8607        state.reviews = vec![ReviewRound {
8608            round: 1,
8609            head: head.clone(),
8610            verified_head: Some(head.clone()),
8611            verified_at: Some(jiff::Timestamp::now()),
8612            reviews: Vec::new(),
8613            e2e: vec![CommandOutcome {
8614                command: format!(
8615                    "CARGO_TARGET_DIR='{}' test -f README.md",
8616                    cache_dir.display()
8617                ),
8618                code: None,
8619                output_tail: "waiting for the shared build cache".to_owned(),
8620                duration_ms: 0,
8621                resource_blocked: true,
8622            }],
8623            fix: None,
8624            blocking: 1,
8625            answered: 1,
8626            expected: 1,
8627            clean: false,
8628            verify_retried: false,
8629            e2e_deferred: false,
8630            e2e_defer_reason: None,
8631            progressed: false,
8632            vote_split: false,
8633            reconsideration: Vec::new(),
8634            verdict: None,
8635        }];
8636
8637        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
8638        let mut runner = Runner {
8639            state,
8640            roles: ResolvedRoles {
8641                implementers: Vec::new(),
8642                judges: Vec::new(),
8643                reviewers: Vec::new(),
8644                fixer: None,
8645                conductor: conductor(),
8646                implementer_roster: Vec::new(),
8647            },
8648            sem: Arc::new(Semaphore::new(1)),
8649            pause: Pause::new(),
8650            interrupt: Pause::new(),
8651        };
8652
8653        // The lease is still held throughout, so this reentry's own retry is
8654        // also contended — proving `review_loop` actually tried again (not
8655        // that it happened to succeed) is what the timestamp comparison
8656        // below is for.
8657        runner.review_loop().await.expect("review_loop");
8658
8659        assert_eq!(
8660            runner.state.reviews.len(),
8661            1,
8662            "no new round was started on top of the unresolved one: {:?}",
8663            runner.state.reviews
8664        );
8665        let last = &runner.state.reviews[0];
8666        assert_eq!(
8667            last.e2e_status(),
8668            E2eStatus::ResourceBlocked,
8669            "still contended: {last:?}"
8670        );
8671        assert!(
8672            last.verified_at.expect("still known") > first_attempt_at,
8673            "review_loop must have actually retried the check, not left it exactly as found"
8674        );
8675        assert_ne!(
8676            runner.state.status,
8677            RunStatus::Blocked,
8678            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
8679            runner.state.status
8680        );
8681
8682        held.release();
8683    }
8684
8685    #[tokio::test]
8686    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
8687        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
8688        let tmp = tempfile::tempdir().expect("tempdir");
8689        let repo = tmp.path().join("repo");
8690        std::fs::create_dir_all(&repo).unwrap();
8691        init_repo(&repo);
8692
8693        let mut config = Config::default();
8694        config.merge.mode = MergeMode::Pr;
8695        config.graph.land = true;
8696        config.graph.land_approval = false;
8697
8698        let mut state = RunState::new(
8699            repo.clone(),
8700            "main".to_owned(),
8701            "deadbeef".to_owned(),
8702            "task".to_owned(),
8703            config,
8704        );
8705        state.candidates = vec![Candidate {
8706            index: 0,
8707            label: 'A',
8708            agent: "alpha".to_owned(),
8709            branch: "does-not-exist".to_owned(),
8710            worktree: repo.clone(),
8711            summary: String::new(),
8712            stat: String::new(),
8713            files: 0,
8714            commits: 0,
8715            empty: false,
8716            failed: None,
8717            verified_noop: None,
8718            duration_ms: 0,
8719            folded: false,
8720        }];
8721        state.tally = Some(Tally {
8722            first_choice: BTreeMap::from([('A', 1)]),
8723            borda: BTreeMap::new(),
8724            winner: 'A',
8725            rankings: 1,
8726            unanimous_initial: true,
8727            deliberated: false,
8728            changed_votes: 0,
8729            unanimous_final: true,
8730            tie_break: None,
8731            judges: 0,
8732            present: 0,
8733            quorum: 0,
8734            met_quorum: true,
8735            uncontested: Some("only candidate A produced a change".to_owned()),
8736        });
8737        state.reviews = vec![ReviewRound {
8738            round: 1,
8739            head: "deadbeef".to_owned(),
8740            verified_head: None,
8741            verified_at: None,
8742            reviews: Vec::new(),
8743            e2e: Vec::new(),
8744            fix: None,
8745            blocking: 0,
8746            answered: 0,
8747            expected: 0,
8748            clean: true,
8749            verify_retried: false,
8750            e2e_deferred: false,
8751            e2e_defer_reason: None,
8752            progressed: false,
8753            vote_split: false,
8754            reconsideration: Vec::new(),
8755            verdict: None,
8756        }];
8757        state.gate = vec![CommandOutcome {
8758            command: "test".to_owned(),
8759            code: Some(0),
8760            output_tail: String::new(),
8761            duration_ms: 0,
8762            resource_blocked: false,
8763        }];
8764        state.gate_ran = true;
8765        // A first pass through `merge` already pushed and opened this pull
8766        // request; `status` is `Landing` because a previous call into `land`
8767        // parked or was interrupted before it reached a terminal outcome.
8768        state.status = RunStatus::Landing;
8769        state.merge = Some(MergeOutcome {
8770            mode: MergeMode::Pr,
8771            ok: true,
8772            detail: "https://example.invalid/x/y/pull/1".to_owned(),
8773        });
8774
8775        // The Landing-resume shortcut calls `run_land` directly rather than
8776        // through `merge`, which is exactly the call site that used to skip
8777        // `settle_questions` - see the fixture below.
8778        ask_test_home();
8779        let store = ask::Questions::open();
8780        let q = ask_open_question(&store, &state.id);
8781
8782        let mut runner = Runner {
8783            state,
8784            roles: ResolvedRoles {
8785                implementers: Vec::new(),
8786                judges: Vec::new(),
8787                reviewers: Vec::new(),
8788                fixer: None,
8789                conductor: conductor(),
8790                implementer_roster: Vec::new(),
8791            },
8792            sem: Arc::new(Semaphore::new(1)),
8793            pause: Pause::new(),
8794            interrupt: Pause::new(),
8795        };
8796
8797        // `execute`, not `merge` directly: the Landing-resume shortcut lives
8798        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
8799        // exactly because `review_loop` would otherwise clobber the marker
8800        // first.
8801        runner.execute().await.expect("execute");
8802
8803        assert_eq!(
8804            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
8805            Some("https://example.invalid/x/y/pull/1"),
8806            "reentry must not push again or open a second pull request over the \
8807             one `land` is already watching"
8808        );
8809        assert_ne!(
8810            runner.state.status,
8811            RunStatus::Landing,
8812            "land could not actually reach the fake pull request, so it must \
8813             have given up rather than left the run silently parked forever"
8814        );
8815        // `land` could not reach the fake pull request, so it gave up into
8816        // `Blocked` - still resumable, so the question must not have been
8817        // swept just because this branch now also calls `settle_questions`.
8818        assert_eq!(runner.state.status, RunStatus::Blocked);
8819        assert!(
8820            store.get(&q.id).unwrap().status.open(),
8821            "Blocked is still alive; settle_questions must have been a no-op here"
8822        );
8823    }
8824
8825    fn state_with_round(round: ReviewRound) -> RunState {
8826        let mut s = RunState::new(
8827            PathBuf::from("/repo"),
8828            "main".to_owned(),
8829            "abc1234".to_owned(),
8830            "add retries".to_owned(),
8831            Config::default(),
8832        );
8833        s.reviews = vec![round];
8834        s
8835    }
8836
8837    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
8838        crate::verdict::Finding {
8839            id: id.to_owned(),
8840            severity,
8841            file: None,
8842            line: None,
8843            title: title.to_owned(),
8844            detail: String::new(),
8845        }
8846    }
8847
8848    #[test]
8849    fn pr_body_names_open_findings_and_declined_ones() {
8850        let round = ReviewRound {
8851            round: 2,
8852            head: "deadbee".to_owned(),
8853            verified_head: None,
8854            verified_at: None,
8855            reviews: vec![ReviewRecord {
8856                attempts: 0,
8857                reviewer: 1,
8858                agent: "alpha".to_owned(),
8859                summary: String::new(),
8860                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
8861                vote: None,
8862                failed: None,
8863                duration_ms: 0,
8864            }],
8865            e2e: vec![CommandOutcome {
8866                command: "cargo test".to_owned(),
8867                code: Some(0),
8868                output_tail: String::new(),
8869                duration_ms: 0,
8870                resource_blocked: false,
8871            }],
8872            verify_retried: false,
8873            e2e_deferred: false,
8874            e2e_defer_reason: None,
8875            fix: Some(FixRecord {
8876                agent: "alpha".to_owned(),
8877                addressed: Vec::new(),
8878                rejected: vec![crate::verdict::Rejection {
8879                    id: "R1-1-1".to_owned(),
8880                    why: "not reachable from any caller".to_owned(),
8881                }],
8882                notes: String::new(),
8883                committed: true,
8884                failed: None,
8885                duration_ms: 0,
8886                continuation: None,
8887            }),
8888            blocking: 0,
8889            answered: 1,
8890            expected: 1,
8891            clean: false,
8892            progressed: true,
8893            vote_split: false,
8894            reconsideration: Vec::new(),
8895            verdict: None,
8896        };
8897        let state = state_with_round(round);
8898        let body = pr_message(&state, 'A').body;
8899
8900        assert!(body.contains("add retries"), "the task must still be there");
8901        assert!(body.contains("R2-1-1"), "{body}");
8902        assert!(body.contains("unused import"), "{body}");
8903        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
8904        assert!(
8905            body.contains("not reachable from any caller"),
8906            "the reason it was declined: {body}"
8907        );
8908    }
8909
8910    #[test]
8911    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
8912        let round = ReviewRound {
8913            round: 1,
8914            head: "deadbee".to_owned(),
8915            verified_head: None,
8916            verified_at: None,
8917            reviews: vec![ReviewRecord {
8918                attempts: 0,
8919                reviewer: 1,
8920                agent: "alpha".to_owned(),
8921                summary: String::new(),
8922                findings: Vec::new(),
8923                vote: None,
8924                failed: None,
8925                duration_ms: 0,
8926            }],
8927            e2e: Vec::new(),
8928            verify_retried: false,
8929            e2e_deferred: false,
8930            e2e_defer_reason: None,
8931            fix: None,
8932            blocking: 0,
8933            answered: 1,
8934            expected: 1,
8935            clean: true,
8936            progressed: false,
8937            vote_split: false,
8938            reconsideration: Vec::new(),
8939            verdict: None,
8940        };
8941        let state = state_with_round(round);
8942        let body = pr_message(&state, 'A').body;
8943        assert!(!body.contains("Open review findings"), "{body}");
8944        assert!(!body.contains("Declined"), "{body}");
8945    }
8946
8947    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
8948        let mut state = RunState::new(
8949            PathBuf::from("/repo"),
8950            "main".to_owned(),
8951            "abc1234".to_owned(),
8952            instruction.to_owned(),
8953            Config::default(),
8954        );
8955        state.candidates.push(Candidate {
8956            index: 0,
8957            label: 'A',
8958            agent: "alpha".to_owned(),
8959            branch: "magi/x/A".to_owned(),
8960            worktree: PathBuf::from("/wt"),
8961            summary: summary.to_owned(),
8962            stat: String::new(),
8963            files: 1,
8964            commits: 1,
8965            empty: false,
8966            failed: None,
8967            verified_noop: None,
8968            folded: false,
8969            duration_ms: 0,
8970        });
8971        state
8972    }
8973
8974    #[test]
8975    fn pr_message_describes_the_change_not_the_task() {
8976        let state = state_with_summary(
8977            "今回やってほしいこと: results projector を直す",
8978            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
8979        );
8980        let m = pr_message(&state, 'A');
8981        assert_eq!(m.title, "fix(web): batch the runs list reads");
8982        assert!(
8983            m.body.starts_with("## Summary\n\n- reads run.json once"),
8984            "{}",
8985            m.body
8986        );
8987        assert!(!m.body.contains("TITLE:"), "{}", m.body);
8988        let task_at = m.body.find("今回やってほしいこと").unwrap();
8989        let details_at = m.body.find("<details>").unwrap();
8990        assert!(
8991            details_at < task_at,
8992            "the task lives inside <details>: {}",
8993            m.body
8994        );
8995        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
8996        assert!(m.body.contains("magi:candidate-a"));
8997    }
8998
8999    #[test]
9000    fn pr_message_falls_back_to_the_task_without_a_title_line() {
9001        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
9002        let m = pr_message(&state, 'A');
9003        assert_eq!(m.title, "add retries");
9004        assert!(
9005            m.body.contains("## Summary\n\n- did some things"),
9006            "{}",
9007            m.body
9008        );
9009
9010        let none = RunState::new(
9011            PathBuf::from("/repo"),
9012            "main".to_owned(),
9013            "abc1234".to_owned(),
9014            "add retries".to_owned(),
9015            Config::default(),
9016        );
9017        let m = pr_message(&none, 'A');
9018        assert_eq!(m.title, "add retries");
9019        assert!(!m.body.contains("## Summary"), "{}", m.body);
9020    }
9021
9022    #[test]
9023    fn pr_message_refuses_the_candidate_commit_subject() {
9024        for bad in [
9025            "TITLE: magi: candidate A (uncommitted work)",
9026            "TITLE: chore: stuff (uncommitted work)",
9027            "TITLE:   ",
9028        ] {
9029            let state = state_with_summary("add retries", bad);
9030            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
9031        }
9032    }
9033
9034    #[test]
9035    fn pr_message_bounds_a_very_long_task_and_title() {
9036        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
9037        let state = state_with_summary(&long, "- nothing");
9038        let m = pr_message(&state, 'A');
9039        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
9040        assert!(!m.title.contains('\n'));
9041
9042        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
9043        let m = pr_message(&state, 'A');
9044        assert!(m.title.starts_with("feat: "));
9045        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
9046        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
9047    }
9048
9049    #[test]
9050    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
9051        // What magi itself writes stays English under any configured language,
9052        // so a future localisation of these headings fails here. (The agents'
9053        // own text is held to English by the prompt only; magi cannot check it.)
9054        let mut state = state_with_summary(
9055            "add retries",
9056            "TITLE: fix(web): batch reads\n- reads run.json once",
9057        );
9058        state.config.graph.language = "ja".to_owned();
9059        let m = pr_message(&state, 'A');
9060        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
9061
9062        // The task is the operator's own text: it goes in untouched, and the
9063        // fallback title (no summary) may be in its language too.
9064        let task = "今回やってほしいこと: results projector を直す";
9065        let mut state = state_with_summary(task, "- no title line");
9066        state.config.graph.language = "ja".to_owned();
9067        let m = pr_message(&state, 'A');
9068        assert_eq!(
9069            m.title,
9070            format!("chore: land candidate A of run {}", state.id)
9071        );
9072        assert!(
9073            m.body.contains(&format!(
9074                "<summary>Original task</summary>\n\n{task}\n\n</details>"
9075            )),
9076            "{}",
9077            m.body
9078        );
9079    }
9080
9081    #[test]
9082    fn pr_message_scrubs_home_paths_and_addresses() {
9083        let state = state_with_summary(
9084            "fix it in /Users/someone/src/x",
9085            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
9086        );
9087        let m = pr_message(&state, 'A');
9088        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
9089            assert!(!m.body.contains(leak), "{}", m.body);
9090        }
9091        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
9092    }
9093
9094    #[test]
9095    fn pr_message_survives_a_task_that_closes_details() {
9096        let state = state_with_summary("a </details> b", "TITLE: fix: x");
9097        let m = pr_message(&state, 'A');
9098        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
9099    }
9100
9101    #[test]
9102    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
9103        let cmd = manual_merge_command(
9104            MergeStyle::Squash,
9105            Path::new("/repo"),
9106            "b",
9107            "fix: \"quoted\" $(x) `y`\n\nbody",
9108        );
9109        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
9110    }
9111
9112    #[test]
9113    fn manual_merge_command_matches_the_configured_style() {
9114        let repo = Path::new("/repo");
9115        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
9116
9117        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
9118        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
9119
9120        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
9121        assert_eq!(
9122            squash,
9123            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
9124             \"Merge magi run 0832 (candidate A)\""
9125        );
9126
9127        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
9128        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
9129    }
9130
9131    #[test]
9132    fn a_nudge_gets_a_quarter_of_the_budget() {
9133        // The judge and implement budgets magi ships with.
9134        assert_eq!(retry_budget(secs(1200), true), secs(300));
9135        assert_eq!(retry_budget(secs(3600), true), secs(900));
9136    }
9137
9138    #[test]
9139    fn a_resent_prompt_keeps_the_whole_budget() {
9140        // The seat kept no context, so the retry is the original job again and
9141        // shortening it would only guarantee a second failure.
9142        assert_eq!(retry_budget(secs(1200), false), secs(1200));
9143        assert_eq!(retry_budget(secs(60), false), secs(60));
9144    }
9145
9146    #[test]
9147    fn the_floor_never_exceeds_the_original_budget() {
9148        // A short configured timeout must not be *raised* by the floor: the
9149        // operator asked for a bound, and a retry may not outlast the attempt
9150        // it is retrying.
9151        assert_eq!(retry_budget(secs(60), true), secs(60));
9152        assert_eq!(retry_budget(secs(480), true), secs(120));
9153        assert_eq!(retry_budget(secs(0), true), secs(0));
9154    }
9155
9156    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
9157        agent::CommandEvidence {
9158            id: "item1".to_owned(),
9159            description: "cargo test".to_owned(),
9160            exit_code,
9161            result_summary: String::new(),
9162            source: "codex".to_owned(),
9163        }
9164    }
9165
9166    #[test]
9167    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
9168        // No evidence is not the same fact as unconfirmed evidence: a
9169        // backend with no adapter, or a reply that ran no commands at all,
9170        // must not be misread as carrying a dangling job.
9171        assert!(!has_unconfirmed_command(&[]));
9172    }
9173
9174    #[test]
9175    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
9176        // Deliberately not a check on the exit code's *value*: a fixer
9177        // legitimately runs something that fails mid-iteration before it
9178        // succeeds, and that must never by itself reopen a valid report.
9179        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
9180        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
9181        assert!(!has_unconfirmed_command(&[
9182            evidence(Some(0)),
9183            evidence(Some(101))
9184        ]));
9185    }
9186
9187    #[test]
9188    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
9189        assert!(has_unconfirmed_command(&[
9190            evidence(Some(0)),
9191            evidence(None)
9192        ]));
9193    }
9194
9195    #[test]
9196    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
9197        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9198        assert_eq!(
9199            verified_noop_claim(true, &[], text).as_deref(),
9200            Some("already fixed by b32cfc4, on main.")
9201        );
9202    }
9203
9204    #[test]
9205    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
9206        // A timeout or a bad exit code reads as the ordinary loss it is,
9207        // whatever the reply's own prose claims.
9208        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9209        assert!(verified_noop_claim(false, &[], text).is_none());
9210    }
9211
9212    #[test]
9213    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
9214        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
9215        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
9216        // A confirmed command alongside the marker is fine.
9217        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
9218    }
9219
9220    #[test]
9221    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
9222        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
9223    }
9224
9225    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
9226    /// pair, in order, labelled A, B, C, ...
9227    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
9228        runner.state.candidates = shape
9229            .iter()
9230            .enumerate()
9231            .map(|(i, &(empty, verified))| Candidate {
9232                index: i,
9233                label: (b'A' + i as u8) as char,
9234                agent: "sonnet".to_owned(),
9235                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
9236                worktree: PathBuf::from(format!("/wt/{i}")),
9237                summary: String::new(),
9238                stat: String::new(),
9239                files: 0,
9240                commits: 0,
9241                empty,
9242                failed: None,
9243                verified_noop: verified.map(str::to_owned),
9244                duration_ms: 0,
9245                folded: false,
9246            })
9247            .collect();
9248    }
9249
9250    #[test]
9251    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
9252        ask_test_home();
9253        let mut runner = runner_at(RunStatus::Implementing);
9254        set_candidates(
9255            &mut runner,
9256            &[
9257                (true, Some("already on main at b32cfc4")),
9258                (true, Some("same fix, see the existing test")),
9259            ],
9260        );
9261
9262        runner
9263            .after_implement()
9264            .expect("a verified no-op is not an error");
9265
9266        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
9267    }
9268
9269    #[test]
9270    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
9271        ask_test_home();
9272        let mut runner = runner_at(RunStatus::Implementing);
9273        // Candidate A declares a verified no-op; candidate B simply wrote
9274        // nothing and said nothing about why. One candidate's claim is not
9275        // the whole run's agreement.
9276        set_candidates(
9277            &mut runner,
9278            &[(true, Some("already on main at b32cfc4")), (true, None)],
9279        );
9280
9281        let err = runner
9282            .after_implement()
9283            .expect_err("an unverified empty candidate must still fail the run");
9284
9285        assert!(
9286            err.to_string().contains("no candidate produced a change"),
9287            "{err}"
9288        );
9289        assert_eq!(runner.state.status, RunStatus::Failed);
9290    }
9291
9292    #[test]
9293    fn after_implement_still_fails_an_ordinary_all_empty_run() {
9294        ask_test_home();
9295        let mut runner = runner_at(RunStatus::Implementing);
9296        set_candidates(&mut runner, &[(true, None), (true, None)]);
9297
9298        let err = runner
9299            .after_implement()
9300            .expect_err("no candidate declared anything; this is an ordinary failure");
9301
9302        assert!(
9303            err.to_string().contains("no candidate produced a change"),
9304            "{err}"
9305        );
9306        assert_eq!(runner.state.status, RunStatus::Failed);
9307    }
9308}