Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::queue;
45use crate::refs;
46use crate::run::{
47    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
48    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord, Handover,
49    JobRecord, JobStatus, Judgement, MergeOutcome, OperatorFixFinding, OperatorFixOutcome,
50    OperatorFixRequest, Origin, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState,
51    RunStatus, SeatHistory, Tally, VoteRecord, tail, write_artifact,
52};
53use crate::verdict::{
54    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
55    ReviewVote, Severity,
56};
57
58/// How much verification output is kept and fed back to the fixer.
59const OUTPUT_TAIL: usize = 8_000;
60
61/// Bytes of a failing command's output kept in an event, so the reason a run
62/// stopped is readable from the report without opening `run.json`.
63const EVENT_OUTPUT_TAIL: usize = 2_000;
64
65/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
66/// command's pid before releasing the build cache's lease.
67const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
68
69/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
70/// command's pid to actually exit before giving up and releasing anyway.
71///
72/// A timeout means the process was asked to die (`kill_on_drop`,
73/// `start_kill`), not that it already has — on Windows in particular that can
74/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
75/// the instant the command returns would let the very next acquirer (this
76/// run's own next round, another run's verification, the janitor's prune)
77/// start touching the same directory while it might still be writing to it,
78/// so this polls the actual pid — real confirmation, not a fixed guess —
79/// until it is gone or this ceiling is reached. It is still not full
80/// process-tree reaping: a grandchild the timed-out process spawned and that
81/// outlives it independently is invisible to a pid check, and continuing to
82/// observe and collect *that* stays a different piece of work with its own
83/// owner. Set generously because the common case returns early the moment
84/// the pid is confirmed gone, not because every timeout pays this in full.
85const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
86
87/// Consecutive review rounds with no tree progress (see
88/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
89/// instead of spending the rest of the round budget.
90///
91/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
92/// legitimately finds nothing left to change (its previous round's fix already
93/// covered it, and this round's reviewers re-raised only nits) looks the same
94/// as one that is spinning, for exactly one round. Two in a row is where the
95/// two stop being distinguishable, and a review round on this workload has
96/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
97/// third attempt at a tree that has not moved twice running is pure cost.
98/// This does not touch `review_rounds` itself, which stays the operator's
99/// call.
100pub(crate) const STAGNANT_LIMIT: usize = 2;
101
102/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
103/// a base that moved before giving up and leaving the run `Blocked` for a
104/// person.
105///
106/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
107/// that keeps moving faster than a run can catch it is not something more
108/// rebasing fixes, it is a person's call. Not the same *number as*
109/// `land_rounds` - this budget is spent before a pull request exists, land's
110/// after - but bounded for the identical reason, so it uses the same
111/// default. Counted across both call sites in [`Runner::finish_after_tally`]
112/// (once before review, once before the gate), because either one finding
113/// the base still moving is the same signal.
114const BASE_SYNC_ROUNDS: usize = 4;
115
116/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
117/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
118/// reply held no [`FixReport`].
119///
120/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
121/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
122/// "I'll pause here until the `cargo make check` background run reports
123/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
124/// No `FixReport` was ever collected from that seat, and the run moved on to
125/// the next review round regardless.
126///
127/// Bounded independently of `review_rounds` and `graph.retries`: this
128/// recovers one seat's missing report mid-round, not a new round of review or
129/// an ordinary parse retry, and must not itself become the unbounded wait the
130/// rest of this module exists to avoid.
131const MAX_FIX_CONTINUATIONS: usize = 2;
132
133/// One queued agent invocation.
134///
135/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
136/// CLI hung up on its own stream is asked again from the same job rather than
137/// rebuilt from scratch. See [`Runner::resume_undelivered`].
138#[derive(Clone)]
139struct SeatJob {
140    spec: AgentSpec,
141    seat: SeatState,
142    cwd: PathBuf,
143    prompt: String,
144    timeout: Duration,
145    allow_write: bool,
146    sessions: bool,
147    artifacts: PathBuf,
148    stem: String,
149    /// The prompt for a seat that has been handed to another roster agent
150    /// (a fresh session): everything the original seat would have
151    /// remembered. `None` when `prompt` already carries it, as the first
152    /// ranking and the implement prompt do. Never a resume-style prompt.
153    handover: Option<String>,
154}
155
156/// How the graph reads one agent invocation.
157///
158/// Quota is split out from an ordinary failure on purpose: a rate-limited call
159/// is known to fail again if retried now, so the retry loop must not spend an
160/// attempt on it. `Dropped` is split out for the opposite reason: unlike
161/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
162/// error JSON, never the agent's answer — a caller that matched only
163/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
164/// left to read that JSON as if it were usable output. `resume_undelivered`
165/// is the only caller that acts on it; everywhere else it is reported like an
166/// ordinary failure.
167enum AgentOutcome {
168    /// A usable output.
169    Ok(AgentOutput),
170    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
171    Quota(AgentOutput),
172    /// The CLI hung up on its own stream after billed work. See
173    /// [`agent::AgentOutput::work_undelivered`].
174    Dropped(AgentOutput),
175    /// Any other failure: a timeout, a bad exit code, an empty reply.
176    Failed(String),
177}
178
179/// A request to park the run at its next node boundary.
180///
181/// Cloning is how the request travels: the loop keeps one handle and hands a
182/// clone to each [`Runner`], and every clone points at the same flag. There
183/// is no channel because there is nothing to send - the only message is
184/// "park", it is idempotent, and a flag cannot be missed by a receiver that
185/// was not listening yet.
186///
187/// The boundary is what makes this cheap. Every node writes the run's state
188/// before the next one starts, and every node skips what is already recorded:
189/// `prep` returns early once candidates exist, `implement` asks only the seats
190/// with nothing on disk, `judge` returns early once judgements exist. So a
191/// parked run resumes into exactly the node it stopped before, and no agent
192/// work is thrown away. Killing the process mid-node, by contrast, loses
193/// whatever the seats in flight had not yet written - which for an implement
194/// wave is an hour of paid work.
195///
196/// A [`Runner`] watches two independent handles of this type - see
197/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
198/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
199/// clone covering the whole daemon's lifetime and is never asked to un-park,
200/// which is correct exactly because nothing is dispatched after it fires.
201/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
202/// that parks for an interrupted task must go on to run other tasks
203/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
204/// reusing the daemon-wide one.
205#[derive(Debug, Clone, Default)]
206pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
207
208impl Pause {
209    /// A pause nobody has asked for yet.
210    #[must_use]
211    pub fn new() -> Self {
212        Self::default()
213    }
214
215    /// Ask the run to park at its next node boundary. Idempotent.
216    pub fn park(&self) {
217        self.0.store(true, Ordering::SeqCst);
218    }
219
220    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
221    /// fold into the run's own `park` event - so an operator reading the run
222    /// later knows this was a deliberate interrupt rather than a shutdown or
223    /// a binary swap. The first reason recorded wins; a park already in
224    /// flight is not relabelled by a second, unrelated request.
225    pub fn park_because(&self, reason: impl Into<String>) {
226        let mut reason_guard = self
227            .1
228            .lock()
229            .unwrap_or_else(std::sync::PoisonError::into_inner);
230        if reason_guard.is_none() {
231            *reason_guard = Some(reason.into());
232        }
233        drop(reason_guard);
234        self.park();
235    }
236
237    /// Has a park been asked for?
238    #[must_use]
239    pub fn parked(&self) -> bool {
240        self.0.load(Ordering::SeqCst)
241    }
242
243    /// Why the park was asked for, when the caller used [`Pause::park_because`].
244    #[must_use]
245    pub fn reason(&self) -> Option<String> {
246        self.1
247            .lock()
248            .unwrap_or_else(std::sync::PoisonError::into_inner)
249            .clone()
250    }
251}
252
253/// Drives one run.
254pub struct Runner {
255    /// Run state; public so the CLI can report on it.
256    pub state: RunState,
257    roles: ResolvedRoles,
258    sem: Arc<Semaphore>,
259    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
260    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
261    /// this is never the same handle as `interrupt`.
262    pause: Pause,
263    /// Set when `magi serve`'s interrupt scheduler wants this specific run
264    /// parked at its next node boundary, to let a task marked
265    /// [`crate::queue::Task::interrupt`] run alone before this one carries
266    /// on. Unlike `pause`, a fresh, unshared handle per run - see
267    /// [`Runner::watch_interrupt`].
268    interrupt: Pause,
269}
270
271/// The commit a run branches from: the base branch as the remote has it.
272///
273/// Two failures this replaces. A run used to branch off `HEAD` and so refused
274/// to start on a dirty tree, which made `magi serve` decline every task for as
275/// long as the operator had work in progress - most of the time. Branching off
276/// the *local* base branch fixed that and introduced a worse one: `land` merges
277/// the winner on GitHub, nothing updates the local ref, and the next run
278/// branches off a base missing everything the previous runs landed. Two tasks
279/// in a row from a phone would have had the second silently re-implementing
280/// against stale code and opening a pull request that reverted the first.
281///
282/// Only refs move here - no checkout, no local branch, no merge - so it is safe
283/// with uncommitted work in the tree. A machine with no network still starts:
284/// the fetch may fail and the local tip is used with a warning, because
285/// refusing to run offline is a worse failure than running against a base the
286/// operator can see for themselves.
287///
288/// One function, called by both entry points. Two answers to "where does a run
289/// branch from" is the kind of drift nobody notices until a diff is wrong.
290/// Bring the local `branch` in line with `<remote>/<branch>` before a review
291/// checks it out.
292///
293/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
294/// anything other than plain `git push` from this checkout (a jj colocated
295/// workspace, another clone) moves only the remote-tracking ref - so the local
296/// one can be a stale placeholder. It moves only when local is behind the remote or is an
297/// empty placeholder that diverged from it; unpushed local work is kept, and a real
298/// divergence is refused rather than guessed at.
299async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
300    let tracking = format!("{remote}/{branch}");
301    let fetched = git::fetch(repo, remote, branch).await;
302    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
303    let local_exists = git::branch_exists(repo, branch).await?;
304    if !fresh {
305        if !local_exists {
306            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
307        }
308        tracing::warn!(
309            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
310        );
311        return Ok(());
312    }
313    let remote_sha = git::rev_parse(repo, &tracking).await?;
314    if !local_exists {
315        git::git(repo, &["branch", branch, &tracking]).await?;
316        return Ok(());
317    }
318    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
319    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
320        return Ok(());
321    }
322    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
323        // Diverged. `reconcile` settles it only when it can prove nothing is
324        // lost: a local tip that is the remote's change rebased is pushed over
325        // it (lease pinned to the tip read here), a tip whose every extra
326        // commit is empty is a placeholder the remote's work replaced, and
327        // anything else is two different changes - a question for a person.
328        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
329            .await?
330        {
331            crate::reconcile::Reconciliation::Pushed => {
332                tracing::warn!(
333                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
334                    short(&local_sha),
335                    short(&remote_sha)
336                );
337                return Ok(());
338            }
339            crate::reconcile::Reconciliation::Placeholder => {}
340            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
341        }
342    }
343    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
344    if !out.ok() {
345        bail!(
346            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
347            short(&local_sha),
348            short(&remote_sha),
349            out.stderr
350        );
351    }
352    tracing::warn!(
353        "local `{branch}` was stale: fast-forwarded {} -> {}",
354        short(&local_sha),
355        short(&remote_sha)
356    );
357    Ok(())
358}
359
360async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
361    let tracking = format!("{remote}/{base_branch}");
362    let fetched = git::fetch(repo, remote, base_branch).await;
363    if let Ok(out) = &fetched
364        && out.ok()
365        && git::rev_exists(repo, &tracking).await
366    {
367        return git::rev_parse(repo, &tracking).await;
368    }
369    let why = match &fetched {
370        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
371        Ok(_) => format!("{remote} has no {base_branch}"),
372        Err(e) => e.to_string(),
373    };
374    tracing::warn!(
375        "could not read {tracking} ({why}); branching off the local \
376         {base_branch} instead, which may be behind"
377    );
378    git::rev_parse(repo, base_branch).await.with_context(|| {
379        format!(
380            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
381             branch that exists"
382        )
383    })
384}
385
386/// Exclusive claim on one run's `magi fix` step, released on drop — including
387/// on an early return or a panic.
388///
389/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
390/// manual `magi fix` invocations against the same run are otherwise
391/// invisible to each other and would race to remove and recreate the same
392/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
393/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
394/// which is only ever reclaimed later, out of band, by
395/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
396/// `magi fix` invocation is not necessarily running under either of those, so
397/// nothing would ever sweep a lock a killed or crashed process left behind.
398/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
399/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
400/// lease use: an unreadable or unparsable pid, or a liveness query the
401/// platform cannot answer, reads as alive and the lock is left in place.
402struct FixClaim {
403    path: PathBuf,
404}
405
406impl FixClaim {
407    fn acquire(dir: &Path) -> Result<Self> {
408        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
409        let path = dir.join("fix.lock");
410        match Self::create(&path) {
411            Ok(claim) => Ok(claim),
412            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
413                if Self::reclaim_if_dead(&path) {
414                    Self::create(&path).with_context(|| format!("lock {}", path.display()))
415                } else {
416                    bail!(
417                        "another `magi fix` is already running for this run ({} exists)",
418                        path.display()
419                    )
420                }
421            }
422            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
423        }
424    }
425
426    fn create(path: &Path) -> std::io::Result<Self> {
427        let mut f = std::fs::OpenOptions::new()
428            .write(true)
429            .create_new(true)
430            .open(path)?;
431        use std::io::Write as _;
432        // Read back by `reclaim_if_dead` on a later, stuck invocation.
433        writeln!(f, "{}", std::process::id())?;
434        Ok(Self {
435            path: path.to_owned(),
436        })
437    }
438
439    /// True if the lock named a process confirmed dead, in which case it was
440    /// also removed. Never true on an unreadable file, an unparsable pid, or
441    /// a liveness query the platform cannot answer — see this type's own doc.
442    fn reclaim_if_dead(path: &Path) -> bool {
443        let dead = std::fs::read_to_string(path)
444            .ok()
445            .and_then(|body| body.trim().parse::<u32>().ok())
446            .is_some_and(|pid| !crate::proc::pid_alive(pid));
447        dead && std::fs::remove_file(path).is_ok()
448    }
449}
450
451impl Drop for FixClaim {
452    fn drop(&mut self) {
453        let _ = std::fs::remove_file(&self.path);
454    }
455}
456
457impl Runner {
458    /// Start a fresh run against `repo`.
459    pub async fn start(
460        repo: &Path,
461        instruction: String,
462        config: Config,
463        origin: Origin,
464    ) -> Result<Self> {
465        Self::start_naming(repo, instruction, "", config, origin).await
466    }
467
468    /// [`Runner::start`] for a queued task: `also_scan` (the task's title) is
469    /// searched for branch and commit references along with the instruction,
470    /// since a task may name the work it is about only in its title.
471    pub async fn start_naming(
472        repo: &Path,
473        instruction: String,
474        also_scan: &str,
475        config: Config,
476        origin: Origin,
477    ) -> Result<Self> {
478        let repo = git::toplevel(repo).await?;
479        let missing = agent::missing_programs(&config.agents);
480        if !missing.is_empty() {
481            bail!(
482                "these agent programs are not on PATH: {}. Fix the roster in \
483                 magi.toml or install them.",
484                missing.join(", ")
485            );
486        }
487        let base_branch = match config.merge.base.clone() {
488            Some(b) => b,
489            None => git::current_branch(&repo)
490                .await?
491                .context("HEAD is detached; set [merge] base in magi.toml")?,
492        };
493        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
494        // Still worth saying out loud. The operator's uncommitted work is not
495        // part of this run, and someone watching a candidate fail to use a
496        // change they just made deserves to know why.
497        if !git::is_clean(&repo).await? {
498            tracing::warn!(
499                "{} has uncommitted changes; they are not part of this run, \
500                 which branches off {base_branch} ({})",
501                repo.display(),
502                &base_commit[..base_commit.len().min(8)]
503            );
504        }
505        let roles = config.resolve_roles()?;
506        let max_parallel = config.graph.max_parallel.max(1);
507        // A task that points at work already in the repository starts from
508        // it; what the repository says about each reference is recorded.
509        let seeds = refs::resolve(
510            &repo,
511            &base_commit,
512            &config.merge.remote,
513            &format!("{also_scan}\n{instruction}"),
514        )
515        .await;
516        refs::plan(&repo, &seeds).await?;
517        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
518        // Recorded before the first save, so a crash right after minting
519        // cannot leave a run with no origin. Legibility only: nothing reads it
520        // to decide anything.
521        state.origin = Some(origin);
522        for seed in &seeds {
523            state.event(
524                "seed",
525                refs::describe(std::slice::from_ref(seed)).unwrap_or_default(),
526            );
527        }
528        state.seeds = seeds;
529        state.event("start", format!("run {} created", state.id));
530        state.save()?;
531        Ok(Self {
532            state,
533            roles,
534            sem: Arc::new(Semaphore::new(max_parallel)),
535            pause: Pause::new(),
536            interrupt: Pause::new(),
537        })
538    }
539
540    /// Open a review-only run against work that already exists on `branch`.
541    ///
542    /// The expensive half of the graph is the implement wave — measured at
543    /// 111 and 134 internal tool-loop turns on this repository, against a
544    /// handful for a judge or a reviewer. The cheap half is worth running on
545    /// hand-written work too, and there was no way to reach it.
546    ///
547    /// No new state and no schema change are needed: a run with **one** viable
548    /// candidate and a tally already decided degrades `execute` to exactly
549    /// review → gate → merge, because `judge` skips a single-candidate field,
550    /// `deliberate` has fewer than two first choices to reconcile, `vote`
551    /// returns early, `tally` is already present and `fold_losers` has no
552    /// losers. Resuming such a run therefore does the right thing as well.
553    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
554        Self::review_taking_over(repo, branch, config, None, origin).await
555    }
556
557    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
558    /// at the same task still has `branch` checked out, its worktree is
559    /// released first if that is safe (see [`crate::handover`]), and the
560    /// review refuses with the reason if it is not. `None` is a hand-run
561    /// review: it has no earlier attempts, so only a worktree of a dead run
562    /// magi recorded itself can be released.
563    pub async fn review_taking_over(
564        repo: &Path,
565        branch: &str,
566        config: Config,
567        takeover: Option<crate::handover::Takeover>,
568        origin: Origin,
569    ) -> Result<Self> {
570        let repo = git::toplevel(repo).await?;
571        let missing = agent::missing_programs(&config.agents);
572        if !missing.is_empty() {
573            bail!(
574                "these agent programs are not on PATH: {}. Fix the roster in \
575                 magi.toml or install them.",
576                missing.join(", ")
577            );
578        }
579        let base_branch = match config.merge.base.clone() {
580            Some(b) => b,
581            None => git::current_branch(&repo)
582                .await?
583                .context("HEAD is detached; set [merge] base in magi.toml")?,
584        };
585        if base_branch == branch {
586            bail!("`{branch}` is the base branch; there is nothing to review against");
587        }
588        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
589
590        let roles = config.resolve_roles()?;
591        let max_parallel = config.graph.max_parallel.max(1);
592        let mut state = RunState::new(
593            repo.clone(),
594            base_branch,
595            base_commit.clone(),
596            String::new(),
597            config,
598        );
599        state.origin = Some(origin);
600
601        // Released before anything else touches the branch: a stale local
602        // branch is moved with `git branch -f`, which git refuses while an
603        // earlier attempt's worktree still has it checked out. Everything
604        // after this point that can fail puts the old run back.
605        // A hand-run review has no task, hence no earlier attempts, but a
606        // worktree of a dead run magi made may still be released.
607        let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
608            earlier: Vec::new(),
609            home: crate::run::home(),
610            choice: None,
611        });
612        let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
613        if let Some(released) = &released {
614            state.event(
615                "release",
616                format!(
617                    "took `{branch}` over from run {}: its worktree was released: {}",
618                    crate::run::short_of(&released.old_id),
619                    released.audit
620                ),
621            );
622        }
623        // The owner's answer to an earlier divergence question is applied
624        // here: after the release (git will not move a checked-out branch)
625        // and before the sync that would otherwise ask again.
626        if let Some(choice) = takeover.choice.as_ref()
627            && let Err(e) =
628                crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
629                    .await
630        {
631            if let Some(released) = &released {
632                released.restore(&repo, branch).await;
633            }
634            return Err(e.context("applying the owner's answer about the diverged branch"));
635        }
636        let opened =
637            Self::open_review(&repo, branch, state, roles, max_parallel, base_commit).await;
638        if opened.is_err()
639            && let Some(released) = &released
640        {
641            released.restore(&repo, branch).await;
642        }
643        opened
644    }
645
646    /// The half of [`Runner::review_taking_over`] that can fail after an
647    /// earlier attempt's worktree was released.
648    async fn open_review(
649        repo: &Path,
650        branch: &str,
651        mut state: RunState,
652        roles: ResolvedRoles,
653        max_parallel: usize,
654        base_commit: String,
655    ) -> Result<Self> {
656        sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
657        // The commit subjects are the closest thing to a task statement that
658        // existing work carries, and the reviewers are told as much.
659        let log = git::log_oneline(repo, &base_commit, branch)
660            .await
661            .unwrap_or_default();
662        let instruction = format!(
663            "Review the work already on branch `{branch}`. There is no task \
664             statement: what the change claims to do is whatever its commits \
665             say.\n\n{}",
666            if log.trim().is_empty() {
667                "(no commit messages)"
668            } else {
669                log.trim()
670            }
671        );
672        state.instruction = instruction;
673        state.reviewed_commits = Some(
674            git::subjects(repo, &base_commit, branch)
675                .await
676                .unwrap_or_default(),
677        );
678
679        // An attached worktree, so the fixer's commits land on the branch under
680        // review rather than on a detached head nobody will look at again.
681        let worktree = state.worktree_root().join("under-review");
682        if let Some(parent) = worktree.parent() {
683            tokio::fs::create_dir_all(parent).await.ok();
684        }
685        let path = worktree.to_string_lossy().to_string();
686        git::git(repo, &["worktree", "add", &path, branch])
687            .await
688            .with_context(|| {
689                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
690            })?;
691
692        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
693            .await
694            .unwrap_or(0);
695        if commits == 0 {
696            git::worktree_remove(repo, &worktree).await.ok();
697            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
698        }
699        let files = git::changed_files(&worktree, &base_commit, "HEAD")
700            .await
701            .map(|f| f.len())
702            .unwrap_or(0);
703        if files == 0
704            && let (Ok(head_tree), Ok(base_tree)) = (
705                git::tree_of(&worktree, "HEAD").await,
706                git::tree_of(&worktree, &base_commit).await,
707            )
708            && head_tree == base_tree
709        {
710            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
711            git::worktree_remove(repo, &worktree).await.ok();
712            bail!(
713                "`{branch}` at {} has a tree identical to base {}; this usually means \
714                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
715                 against `{}/{branch}`) rather than an empty change",
716                short(&head),
717                short(&base_commit),
718                state.config.merge.remote
719            );
720        }
721        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
722            .await
723            .unwrap_or_default();
724
725        state.candidates.push(Candidate {
726            index: 0,
727            label: 'A',
728            // Not an agent id on purpose: nothing in the roster wrote this, and
729            // the stats tables must not credit anyone with a win for it.
730            agent: EXISTING_BRANCH.to_owned(),
731            branch: branch.to_owned(),
732            worktree,
733            summary: String::new(),
734            stat,
735            files,
736            commits,
737            empty: false,
738            failed: None,
739            verified_noop: None,
740            duration_ms: 0,
741            folded: false,
742        });
743        state.tally = Some(Tally {
744            first_choice: BTreeMap::from([('A', 0)]),
745            borda: BTreeMap::new(),
746            winner: 'A',
747            rankings: 0,
748            unanimous_initial: false,
749            deliberated: false,
750            changed_votes: 0,
751            unanimous_final: false,
752            tie_break: None,
753            // No panel sat, so no quorum applies. Zero judges is the correct
754            // number for work that never competed, and must not be reported as
755            // a collapsed panel.
756            judges: 0,
757            present: 0,
758            quorum: 0,
759            met_quorum: true,
760            uncontested: Some("review-only run: nothing competed".to_owned()),
761        });
762        state.status = RunStatus::Reviewing;
763        state.event(
764            "start",
765            format!(
766                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
767                state.id
768            ),
769        );
770        state.save()?;
771        Ok(Self {
772            state,
773            roles,
774            sem: Arc::new(Semaphore::new(max_parallel)),
775            pause: Pause::new(),
776            interrupt: Pause::new(),
777        })
778    }
779
780    /// Reopen an existing run.
781    pub fn resume(id: &str) -> Result<Self> {
782        let state = RunState::load(id)?;
783        if let Some(to) = &state.released_to {
784            bail!(
785                "run {} cannot be resumed: its worktree was released to run {}",
786                state.short(),
787                crate::run::short_of(to)
788            );
789        }
790        let roles = state.config.resolve_roles()?;
791        let max_parallel = state.config.graph.max_parallel.max(1);
792        Ok(Self {
793            state,
794            roles,
795            sem: Arc::new(Semaphore::new(max_parallel)),
796            pause: Pause::new(),
797            interrupt: Pause::new(),
798        })
799    }
800
801    /// Walk the graph to a terminal state, skipping nodes already recorded.
802    ///
803    /// Every way a run is driven - the queue loop, `magi run`, a resume from
804    /// the phone - ends here, so this is the one place a run that ended
805    /// Blocked / Stalled / Failed, or died with an error, is announced to the
806    /// notification centre. Best-effort: see [`crate::notices::raise`].
807    pub async fn execute(&mut self) -> Result<()> {
808        let result = self.execute_graph().await;
809        self.mark_driver_exited();
810        let ended = if result.is_err() {
811            Some(crate::notices::run_stopped(&self.state.id, &self.state))
812        } else {
813            crate::notices::run_ended(&self.state)
814        };
815        if let Some(notice) = ended {
816            crate::notices::raise(notice);
817        }
818        result
819    }
820
821    /// Record that this process no longer drives the run, so its pid (a
822    /// daemon's outlives the run) is not read as a live driver.
823    ///
824    /// Written onto the record as it is on disk, never this copy: another
825    /// process may have resumed the run (recording its own pid and clearing
826    /// the flag) or released its worktree since this copy was read, and
827    /// saving over that would mark a running driver dead. Only a record still
828    /// naming this process as the driver is touched.
829    fn mark_driver_exited(&mut self) {
830        self.state.driver_exited = true;
831        let pid = std::process::id();
832        let Ok(mut disk) = RunState::load(&self.state.id) else {
833            return;
834        };
835        if disk.released_to.is_some() || disk.driver_pid != Some(pid) || disk.driver_exited {
836            return;
837        }
838        disk.driver_exited = true;
839        if let Err(e) = disk.save() {
840            tracing::warn!("could not record that run {} stopped: {e:#}", self.state.id);
841        }
842    }
843
844    async fn execute_graph(&mut self) -> Result<()> {
845        // Moving again, so it is no longer parked. Set before the walk rather
846        // than in `resume`, so every way of re-entering the graph clears it
847        // and a card cannot claim a run is waiting to be resumed while the
848        // agents are already working.
849        self.state.parked = false;
850        // Any seat this state still lists as answering belongs to whatever
851        // process last drove this run — this one included, if it crashed
852        // mid-wave. Cleared and flushed immediately, before anything else
853        // runs, so a resume can never show a seat as live when nothing is
854        // asking it anything yet; the node that actually dispatches the next
855        // wave repopulates it.
856        self.state.clear_active();
857        // Recorded in the same spot, and flushed together with the clear
858        // above: this is the pid a reader checks (`RunState::liveness`) when
859        // no daemon claim exists to answer "is a process still driving this
860        // run" — a plain `magi run` / `magi review` typed into a terminal
861        // claims nothing there. Always overwritten, never only-if-absent, so
862        // a resumed run's stale pid from a previous, possibly-dead process
863        // can never survive into this one's own report. Unlike
864        // `clear_active`, this changes on every single `execute()` call, so
865        // the save below is now unconditional rather than only-if-cleared.
866        //
867        // `driver_started_at` is recorded in the same breath, from this same
868        // pid, so `liveness` can tell a live pid that is genuinely still us
869        // apart from one the OS has since handed to an unrelated process —
870        // see that field's own doc for why the pid alone is not enough.
871        // A resume that raced a takeover: the record on disk says the worktree
872        // was handed to a later run after this copy was read. Saving over it
873        // would erase that and drive a run with nothing to run in.
874        if let Ok(disk) = RunState::load(&self.state.id)
875            && let Some(to) = &disk.released_to
876        {
877            bail!(
878                "run {} cannot continue: its worktree was released to run {}",
879                self.state.short(),
880                crate::run::short_of(to)
881            );
882        }
883        let pid = std::process::id();
884        self.state.driver_pid = Some(pid);
885        self.state.driver_started_at = crate::proc::process_started_at(pid);
886        self.state.driver_exited = false;
887        self.state.save()?;
888        // A run that already lost its quorum never resumes into the verdict
889        // machinery: `deliberate` and `vote` would otherwise clobber the
890        // stalled marker back to Voting and the run would keep going past a
891        // verdict that is no longer trustworthy. Everything already recorded is
892        // kept, so the run stays resumable (or foldable) for a human to pick up.
893        //
894        // On --resume the run gets one chance to repair itself: the seats a
895        // rate limit took out are re-asked. If their quota has since reset and
896        // the quorum is restored, the run picks up and finishes; otherwise it
897        // stays stale and still-resumable for a later retry. If it does not
898        // recover, the returned status stays `Stalled` and nothing was
899        // clobbered (the recovery only mutates entries for the lost seats).
900        if self.state.status == RunStatus::Stalled {
901            if self.recover_stall().await? {
902                self.finish_after_tally().await?;
903            } else {
904                // Still below quorum: persist the marker and stay resumable.
905                self.state.save()?;
906            }
907            return Ok(());
908        }
909        // A run parked inside `land` - watching CI, mid fix-round, or
910        // waiting on the owner's merge approval - resumes directly into it,
911        // never back through `prep`. Everything before `merge` already
912        // concluded; that is the only way `status` reaches `Landing` in the
913        // first place. Re-walking `review_loop` first would also be actively
914        // wrong: its own status recomputation (see its doc) treats any
915        // clean round as reason to set `status` to `Gating`, which would
916        // clobber this marker before `merge` ever ran, and this run would
917        // never find its way back into `land` at all.
918        if self.state.status == RunStatus::Landing {
919            self.run_land().await?;
920            // `run_land` may have settled the run right here - CI came back
921            // green and the PR merged, say - without ever passing back
922            // through `merge`'s own trailing call. Whatever it left `status`
923            // as is what this has to read.
924            self.settle_questions();
925            return Ok(());
926        }
927        self.prep().await?;
928        if self.park_here()? {
929            return Ok(());
930        }
931        self.advise().await?;
932        if self.park_here()? {
933            return Ok(());
934        }
935        self.implement().await?;
936        if self.park_here()? {
937            return Ok(());
938        }
939        // `after_implement` already saved the state and settled any open
940        // questions when it set this; nothing later in the graph has
941        // anything to judge.
942        if self.state.status == RunStatus::VerifiedNoop {
943            return Ok(());
944        }
945        self.judge().await?;
946        if self.park_here()? {
947            return Ok(());
948        }
949        self.deliberate().await?;
950        if self.park_here()? {
951            return Ok(());
952        }
953        self.vote().await?;
954        if self.park_here()? {
955            return Ok(());
956        }
957        self.tally()?;
958        // A verdict that lost its quorum is not trustworthy: do not review,
959        // gate, or merge on it. Everything already done is kept, so the run
960        // stays resumable (or foldable); the human can replace the agent that
961        // ran out of quota and pick it up.
962        if self.state.status == RunStatus::Stalled {
963            // Persist the stalled marker now — the normal end-of-execute save
964            // below is below this early return, and without it a resumed run
965            // would reload a pre-tally status and keep going.
966            self.state.save()?;
967            return Ok(());
968        }
969        self.finish_after_tally().await?;
970        Ok(())
971    }
972
973    /// Park here if asked to, recording it in the run's own timeline.
974    ///
975    /// Returns whether the caller should stop walking the graph. The state is
976    /// saved either way by the node that just finished; this adds the event so
977    /// the operator's card says why a run that is neither finished nor moving
978    /// is sitting where it is.
979    fn park_here(&mut self) -> Result<bool> {
980        // Either handle asking is enough - see `Pause`'s own doc for why
981        // they are never the same one. `interrupt` is checked second so a
982        // reason it carries is preferred in the message below over a plain
983        // shutdown park racing it at the same boundary.
984        if !self.pause.parked() && !self.interrupt.parked() {
985            return Ok(false);
986        }
987        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
988            Some(reason) => format!(
989                "parked after `{}` ({reason}) — resume to carry on from here",
990                self.state.status.as_str()
991            ),
992            None => format!(
993                "parked after `{}` — resume to carry on from here",
994                self.state.status.as_str()
995            ),
996        };
997        self.state.event("park", why);
998        self.state.parked = true;
999        self.state.save()?;
1000        Ok(true)
1001    }
1002
1003    /// Hand the runner the pause `magi serve`'s own shutdown watches.
1004    pub fn on_pause(&mut self, pause: Pause) {
1005        self.pause = pause;
1006    }
1007
1008    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
1009    /// scheduler asking this one run - and no other - to park so a task
1010    /// marked [`crate::queue::Task::interrupt`] can run alone. See
1011    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
1012    /// handle.
1013    pub fn watch_interrupt(&mut self, pause: Pause) {
1014        self.interrupt = pause;
1015    }
1016
1017    /// Abandon this run's own open questions, once `status` has actually
1018    /// settled rather than merely paused.
1019    ///
1020    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
1021    /// either back up with the candidates, the review round and the seat
1022    /// sessions already on disk, so a question an implementer asked mid-round
1023    /// may still get a real answer read by a real resume. Only the statuses
1024    /// `resumable` excludes are actually final: the run merged, it reached
1025    /// `Ready` with nothing left to do, it failed outright with no
1026    /// established point to continue from, or every candidate agreed, with
1027    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
1028    /// every one of those the seat that asked is gone for good, exactly like
1029    /// the run being deleted under `magi run rm` - so the same cleanup
1030    /// applies, worded for what actually happened instead of "the run was
1031    /// deleted".
1032    ///
1033    /// Best-effort and silent on success: called from every place `status`
1034    /// can land on one of those three, including ones a resumed run revisits,
1035    /// so it must cost nothing when there was nothing open to begin with.
1036    fn settle_questions(&mut self) {
1037        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
1038            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
1039        }
1040    }
1041
1042    /// The tail of the graph after a trustworthy tally: fold losers, review,
1043    /// gate, merge, and persist.
1044    async fn finish_after_tally(&mut self) -> Result<()> {
1045        self.fold_losers().await?;
1046        // Before review starts, and again right before the gate: a run's
1047        // review rounds can themselves take long enough for the base to move
1048        // a second time, and the gate is the one node whose "green" gets
1049        // acted on.
1050        self.sync_to_base().await?;
1051        if self.state.status == RunStatus::AlreadyInBase {
1052            return Ok(());
1053        }
1054        self.review_loop().await?;
1055        self.sync_to_base().await?;
1056        if self.state.status == RunStatus::AlreadyInBase {
1057            return Ok(());
1058        }
1059        self.gate().await?;
1060        self.merge().await?;
1061        self.state.save()?;
1062        Ok(())
1063    }
1064
1065    // ---------------------------------------------------------------- prep
1066
1067    async fn prep(&mut self) -> Result<()> {
1068        if !self.state.candidates.is_empty() {
1069            return Ok(());
1070        }
1071        self.state.status = RunStatus::Prep;
1072        let repo = self.state.repo.clone();
1073        let base = self.state.base_commit.clone();
1074        let plan = refs::plan(&repo, &self.state.seeds).await?;
1075        let start = plan.start.clone().unwrap_or_else(|| base.clone());
1076        let root = self.state.worktree_root();
1077        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
1078
1079        // The hook is the write-time half of the blindness contract; the
1080        // presentation filter in `blind` is the half that cannot be bypassed.
1081        let hooks_dir = self.state.dir().join("hooks");
1082        if self.state.config.blind.commit_msg_hook {
1083            std::fs::create_dir_all(&hooks_dir)
1084                .with_context(|| format!("create {}", hooks_dir.display()))?;
1085            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
1086            let path = hooks_dir.join("commit-msg");
1087            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
1088            make_executable(&path)?;
1089            // Ref-counted rather than a plain idempotent set: with more than
1090            // one run able to be in flight in the same repository at once
1091            // (see `Config::daemon.max_concurrent_runs`), a bare "already
1092            // true?" check cannot tell "another run of mine still needs
1093            // this" from "nobody does", and the run that happens to finish
1094            // first would disable the hook out from under a sibling still
1095            // relying on it.
1096            git::acquire_worktree_config(&repo).await?;
1097            self.state.enabled_worktree_config = true;
1098        }
1099
1100        for (index, (spec, label)) in self
1101            .roles
1102            .implementers
1103            .clone()
1104            .into_iter()
1105            .zip(labels)
1106            .enumerate()
1107        {
1108            let branch = self.state.branch_for(label);
1109            let worktree = root.join(format!("cand-{label}"));
1110            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
1111            if self.state.config.blind.commit_msg_hook {
1112                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
1113            }
1114            git::local_exclude(&worktree, "/.magi/").await?;
1115            for pick in &plan.picks {
1116                if let Err(e) = git::cherry_pick(&worktree, pick).await {
1117                    self.state.status = RunStatus::Blocked;
1118                    self.state
1119                        .event("prep", format!("cannot apply referenced commit: {e}"));
1120                    self.state.save()?;
1121                    return Err(e);
1122                }
1123            }
1124            self.state.candidates.push(Candidate {
1125                index,
1126                label,
1127                agent: spec.id.clone(),
1128                branch,
1129                worktree,
1130                summary: String::new(),
1131                stat: String::new(),
1132                files: 0,
1133                commits: 0,
1134                empty: false,
1135                failed: None,
1136                verified_noop: None,
1137                duration_ms: 0,
1138                folded: false,
1139            });
1140        }
1141
1142        for j in 1..=self.roles.judges.len() {
1143            let wt = root.join(format!("judge-{j}"));
1144            if !wt.exists() {
1145                git::worktree_add_detached(&repo, &wt, &base).await?;
1146            }
1147        }
1148
1149        // Disposable, detached checkouts for the design-deliberation stage's
1150        // advisor seats — the same shape as the judges' above, at the same
1151        // base commit, since advisors also only ever read. Sized off the
1152        // configured count directly rather than a resolved roster: unlike
1153        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
1154        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
1155        // `prep` has no `ResolvedRoles` field to read a count from here.
1156        if self.state.config.graph.advise {
1157            for k in 1..=self.state.config.graph.advisors {
1158                let wt = root.join(format!("advisor-{k}"));
1159                if !wt.exists() {
1160                    git::worktree_add_detached(&repo, &wt, &base).await?;
1161                }
1162            }
1163        }
1164
1165        // A judge cannot tell it is looking at its own patch — the seats keep
1166        // separate conversations — but a panel that shares agents with the
1167        // field is less independent than it looks, and that is worth saying out
1168        // loud once per run rather than leaving it in the config.
1169        let authors: Vec<&str> = self
1170            .roles
1171            .implementers
1172            .iter()
1173            .map(|a| a.id.as_str())
1174            .collect();
1175        let overlap: Vec<String> = self
1176            .roles
1177            .judges
1178            .iter()
1179            .enumerate()
1180            .filter(|(_, j)| authors.contains(&j.id.as_str()))
1181            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
1182            .collect();
1183        if !overlap.is_empty() {
1184            let note = format!(
1185                "{} also authored a candidate; blind, but the panel is less \
1186                 independent than {} distinct agents would be",
1187                overlap.join(", "),
1188                self.roles.judges.len()
1189            );
1190            self.state.event("prep", note);
1191        }
1192
1193        self.state.event(
1194            "prep",
1195            format!(
1196                "{} candidates, {} judges, base {} ({})",
1197                self.state.candidates.len(),
1198                self.roles.judges.len(),
1199                &self.state.base_commit[..7.min(self.state.base_commit.len())],
1200                self.state.base_branch
1201            ),
1202        );
1203        self.state.status = RunStatus::Implementing;
1204        self.state.save()?;
1205        Ok(())
1206    }
1207
1208    // -------------------------------------------------------------- advise
1209
1210    /// The design-deliberation stage: independent, read-only advisor seats
1211    /// each sketch a design before any implementer touches the repository,
1212    /// and (when at least one produced a usable proposal) a synthesis seat
1213    /// blends them into a brief `implement` carries in every candidate's
1214    /// prompt.
1215    ///
1216    /// `[graph] advise` is the on/off switch, on by default; `[graph]
1217    /// advisors` is the proposal count. Everything here is best-effort and
1218    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
1219    /// that cannot reach quota, or a synthesis seat that produced nothing
1220    /// usable all leave `implement` exactly as it was before this stage
1221    /// existed — the task instruction alone — rather than failing the whole
1222    /// competition over an enrichment stage. Every outcome is still recorded
1223    /// as an event, so a run that got nothing from this stage says why.
1224    ///
1225    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
1226    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
1227    /// resumed run whose stage failed would re-run it, and re-spend the
1228    /// agent calls, on every reentry before `implement`.
1229    ///
1230    /// Also skipped once any candidate shows implementation progress — the
1231    /// exact predicate `implement` itself uses to decide a candidate is no
1232    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
1233    /// is not enough: a run created by an older binary that predates this
1234    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
1235    /// an already-`Implementing`-or-later run under this build would
1236    /// otherwise walk straight back through `prep` (a no-op once candidates
1237    /// exist) into this node and spawn every advisor seat against worktrees
1238    /// `prep` never recreated — after implementation has already started,
1239    /// which is exactly the invariant this stage exists to guarantee.
1240    async fn advise(&mut self) -> Result<()> {
1241        let implement_untouched = self
1242            .state
1243            .candidates
1244            .iter()
1245            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
1246        if !self.state.config.graph.advise || self.state.advise_attempted {
1247            return Ok(());
1248        }
1249        if !implement_untouched {
1250            self.state.event(
1251                "advise",
1252                "skipping the design-deliberation stage: at least one \
1253                 candidate already shows implementation progress, so this \
1254                 run is past the point the stage exists to run before"
1255                    .to_owned(),
1256            );
1257            self.state.advise_attempted = true;
1258            self.state.save()?;
1259            return Ok(());
1260        }
1261        let run_id = self.state.id.clone();
1262        let prompts = self.state.config.prompts.clone();
1263        let instruction = self.state.instruction.clone();
1264        let language = self.state.config.graph.language.clone();
1265        let root = self.state.worktree_root();
1266        let n = self.state.config.graph.advisors;
1267        let where_recorded = self.state.dir().join("run.json");
1268
1269        let seats = match self.state.config.advisors() {
1270            Ok(seats) if !seats.is_empty() => seats,
1271            Ok(_) => {
1272                self.state.event(
1273                    "advise",
1274                    format!(
1275                        "[graph] advisors is 0; skipping the design-deliberation \
1276                         stage and continuing without a synthesis brief (see {})",
1277                        where_recorded.display()
1278                    ),
1279                );
1280                self.state.advise_attempted = true;
1281                self.state.save()?;
1282                return Ok(());
1283            }
1284            Err(e) => {
1285                self.state.event(
1286                    "advise",
1287                    format!(
1288                        "could not resolve advisor seats ({e:#}); continuing \
1289                         without a design-deliberation brief (see {})",
1290                        where_recorded.display()
1291                    ),
1292                );
1293                self.state.advise_attempted = true;
1294                self.state.save()?;
1295                return Ok(());
1296            }
1297        };
1298
1299        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1300        let artifacts = agent::artifacts_dir(&self.state.dir());
1301        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
1302
1303        let mut jobs = Vec::new();
1304        for (i, spec) in seats.iter().cloned().enumerate() {
1305            let seat_key = format!("advisor-{}", i + 1);
1306            let seat = self.seat(&seat_key, &spec.id);
1307            jobs.push(SeatJob {
1308                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
1309                spec,
1310                seat,
1311                cwd: worktrees[i % worktrees.len()].clone(),
1312                timeout,
1313                allow_write: false,
1314                sessions: false,
1315                artifacts: artifacts.clone(),
1316                stem: seat_key,
1317                handover: None,
1318            });
1319        }
1320
1321        self.state.event(
1322            "advise",
1323            format!(
1324                "{} advisor seat(s) sketching a design in parallel",
1325                jobs.len()
1326            ),
1327        );
1328        let mut quota_losses = Vec::new();
1329        let cache = self.state.config.cache_dir();
1330        let ctx = WaveCtx {
1331            carry_seats: false,
1332            run: &run_id,
1333            node: "advise",
1334            prompts: &prompts,
1335            cache: cache.as_deref(),
1336            round: None,
1337        };
1338        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
1339        let results = ask_json_wave::<Proposal>(
1340            jobs,
1341            Arc::clone(&self.sem),
1342            self.state.config.graph.retries,
1343            &advisor_roster,
1344            &ctx,
1345            &mut quota_losses,
1346            &mut self.state,
1347            &|p: &Proposal| p.validate(),
1348        )
1349        .await;
1350        self.state.quota.extend(quota_losses);
1351
1352        let mut records = Vec::with_capacity(results.len());
1353        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
1354            let agent_id = seat.agent.clone();
1355            self.state.seats.insert(seat.key.clone(), seat);
1356            match res {
1357                Ok((proposal, out)) => {
1358                    self.state
1359                        .event("advise", format!("advisor-{} proposed a design", i + 1));
1360                    records.push(advise::AdvisorRecord::proposed(
1361                        i + 1,
1362                        agent_id,
1363                        proposal,
1364                        out.duration_ms,
1365                    ));
1366                }
1367                Err(e) => {
1368                    self.state.event(
1369                        "advise",
1370                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
1371                    );
1372                    records.push(advise::AdvisorRecord::failed(
1373                        i + 1,
1374                        agent_id,
1375                        e.to_string(),
1376                    ));
1377                }
1378            }
1379        }
1380
1381        let mut advice = advise::Advice {
1382            records,
1383            synthesis: None,
1384        };
1385        if advice.proposals().is_empty() {
1386            self.state.event(
1387                "advise",
1388                "no advisor produced a usable proposal; continuing without a \
1389                 synthesis brief"
1390                    .to_owned(),
1391            );
1392        } else {
1393            match self
1394                .synthesize_brief(
1395                    &advice,
1396                    &instruction,
1397                    &language,
1398                    &worktrees[0],
1399                    &artifacts,
1400                    &run_id,
1401                    &prompts,
1402                    cache.as_deref(),
1403                )
1404                .await
1405            {
1406                Ok(Some(text)) => {
1407                    self.state.event(
1408                        "advise",
1409                        "synthesized a design brief for the implementer".to_owned(),
1410                    );
1411                    advice.synthesis = Some(text);
1412                }
1413                Ok(None) => {
1414                    self.state.event(
1415                        "advise",
1416                        "the synthesis seat produced nothing usable; continuing \
1417                         without a design brief"
1418                            .to_owned(),
1419                    );
1420                }
1421                Err(e) => {
1422                    self.state.event(
1423                        "advise",
1424                        format!("could not synthesize a design brief: {e:#}"),
1425                    );
1426                }
1427            }
1428        }
1429        advise::apply_reflection(&mut advice);
1430
1431        self.state.advice = Some(advice);
1432        self.state.advise_attempted = true;
1433        self.state.save()?;
1434        Ok(())
1435    }
1436
1437    /// The synthesis seat: reads every advisor's proposal and blends them
1438    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1439    /// out of [`Runner::advise`] only for readability — it is not called
1440    /// anywhere else.
1441    ///
1442    /// Picked the same way [`crate::talk`]'s standing conversation and
1443    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
1444    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
1445    /// order (a claude seat, else the first runnable agent in roster order)
1446    /// used when that field is unset — see `[roles] synthesizer`'s own doc
1447    /// in [`crate::config`] for why a dedicated field exists here at all.
1448    #[allow(clippy::too_many_arguments)]
1449    async fn synthesize_brief(
1450        &mut self,
1451        advice: &advise::Advice,
1452        instruction: &str,
1453        language: &str,
1454        cwd: &Path,
1455        artifacts: &Path,
1456        run_id: &str,
1457        prompts: &Prompts,
1458        cache: Option<&Path>,
1459    ) -> Result<Option<String>> {
1460        let chain = agent::pick_chain(
1461            &self.state.config.agents,
1462            self.state.config.roles.synthesizer.as_ref(),
1463            &agent::installed,
1464            "synthesizer",
1465        )?;
1466        let proposals = advice.proposals();
1467        let mut prompt = prompt::with_overlay(
1468            prompt::synthesize_brief(instruction, &proposals, language),
1469            prompts.overlay("advise"),
1470        );
1471        if cache.is_some() {
1472            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1473            // below — see `prompt::build_cache_note`'s doc for why telling a
1474            // read-only seat to build through the shared cache is exactly how
1475            // a sandbox's write refusal gets misread as a defect.
1476            prompt.push('\n');
1477            prompt.push_str(&prompt::build_cache_note("advise", false));
1478        }
1479        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1480        // Each id is tried once, in order; a quota hit, error or unusable
1481        // answer moves to the next. The seat is single-turn (`sessions:
1482        // false`) and the prompt is the whole context, so a fallback agent
1483        // needs nothing carried over.
1484        let mut last = None;
1485        for (n, spec) in chain.iter().enumerate() {
1486            if n > 0 {
1487                self.state
1488                    .event("advise", format!("synthesis falling back to {}", spec.id));
1489            }
1490            let mut seat = self.seat("advise-synthesis", &spec.id);
1491            let outcome = agent::invoke(
1492                spec,
1493                &mut seat,
1494                &Invocation {
1495                    cwd,
1496                    prompt: &prompt,
1497                    timeout,
1498                    allow_write: false,
1499                    sessions: false,
1500                    artifacts,
1501                    stem: &if n == 0 {
1502                        "advise-synthesis".to_owned()
1503                    } else {
1504                        format!("advise-synthesis-{}", spec.id)
1505                    },
1506                    run: run_id,
1507                    node: "advise",
1508                    cache_dir: None,
1509                    attachments: &[],
1510                    writable: &[],
1511                },
1512            )
1513            .await;
1514            if outcome.is_ok() {
1515                self.state.seats.insert(seat.key.clone(), seat);
1516            }
1517            let advance = agent::chain_advances(&outcome);
1518            last = Some(outcome);
1519            if !advance {
1520                break;
1521            }
1522        }
1523        // Exhausted: the last attempt's result is what a single failed seat
1524        // would have produced.
1525        let out = last.expect("a chain holds at least one agent")?;
1526        if !out.usable() {
1527            return Ok(None);
1528        }
1529        let text =
1530            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1531        Ok((!text.trim().is_empty()).then_some(text))
1532    }
1533
1534    // ----------------------------------------------------------- implement
1535
1536    async fn implement(&mut self) -> Result<()> {
1537        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1538        // agent files with `magi task add` name the run that paid for it. The
1539        // prompt overlay is cloned alongside it because the waves borrow it
1540        // while `self` is mutably borrowed by the node's own bookkeeping.
1541        let run_id = self.state.id.clone();
1542        let prompts = self.state.config.prompts.clone();
1543        let todo: Vec<usize> = self
1544            .state
1545            .candidates
1546            .iter()
1547            .enumerate()
1548            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1549            .map(|(i, _)| i)
1550            .collect();
1551        if todo.is_empty() {
1552            return self.after_implement();
1553        }
1554        self.state.status = RunStatus::Implementing;
1555
1556        let language = self.state.config.graph.language.clone();
1557        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1558        let sessions = self.state.config.graph.sessions;
1559        let artifacts = agent::artifacts_dir(&self.state.dir());
1560        // The design-deliberation stage's blended brief, when `advise` found
1561        // one — carried into every implementer's prompt the same way
1562        // regardless of which candidate it is.
1563        let brief = self
1564            .state
1565            .advice
1566            .as_ref()
1567            .and_then(|a| a.synthesis.as_deref())
1568            .map(str::to_owned);
1569        let attachments = self.state.attachments.clone();
1570
1571        let mut jobs = Vec::new();
1572        for &i in &todo {
1573            let (index, label, worktree) = {
1574                let c = &self.state.candidates[i];
1575                (c.index, c.label, c.worktree.clone())
1576            };
1577            let spec = self.roles.implementers[index].clone();
1578            let seat_key = format!("impl-{label}");
1579            let seat = self.seat(&seat_key, &spec.id);
1580            let instruction = seeded_instruction(&self.state);
1581            jobs.push(SeatJob {
1582                spec,
1583                seat,
1584                prompt: prompt::implement(
1585                    &instruction,
1586                    &worktree.to_string_lossy(),
1587                    &language,
1588                    brief.as_deref(),
1589                    &attachments,
1590                ),
1591                cwd: worktree,
1592                timeout,
1593                allow_write: true,
1594                sessions,
1595                artifacts: artifacts.clone(),
1596                stem: format!("impl-{label}"),
1597                handover: None,
1598            });
1599        }
1600
1601        self.state.event(
1602            "implement",
1603            format!("{} candidates in parallel", jobs.len()),
1604        );
1605        // Kept so a seat whose CLI hung up can be asked again from the same
1606        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
1607        // can update a seat's own entry once a fallback agent takes it over —
1608        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
1609        // whichever agent actually answered, not the one that quota'd out.
1610        let mut sent = jobs.clone();
1611        let cache = self.state.config.cache_dir();
1612        let ctx = WaveCtx {
1613            carry_seats: false,
1614            run: &run_id,
1615            node: "implement",
1616            prompts: &prompts,
1617            cache: cache.as_deref(),
1618            round: None,
1619        };
1620        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1621        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1622            .await;
1623        self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
1624            .await;
1625        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1626            .await;
1627
1628        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1629            let seat_key = seat.key.clone();
1630            // A quota fallback (`resume_seat_handovers`) may have handed this
1631            // seat to a different agent than the one `prep` recorded on the
1632            // candidate; the stats tables and any later fixer-defaults-to-
1633            // winner's-author lookup must credit whoever actually answered —
1634            // unless every fallback also quota'd out, in which case nobody
1635            // actually answered and crediting the last agent tried would
1636            // erase every earlier agent's own quota loss from the stats
1637            // tables instead of just this one seat's.
1638            let agent = seat.agent.clone();
1639            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
1640            self.state.seats.insert(seat.key.clone(), seat);
1641            let label = self.state.candidates[i].label;
1642            let worktree = self.state.candidates[i].worktree.clone();
1643            let base = self.state.base_commit.clone();
1644
1645            let (summary, duration, failed, verified_claim) = match out {
1646                AgentOutcome::Ok(o) => {
1647                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1648                    let failed = (!o.usable()).then(|| {
1649                        if o.timed_out {
1650                            "agent timed out".to_owned()
1651                        } else {
1652                            format!("agent exited with {:?}", o.exit_code)
1653                        }
1654                    });
1655                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
1656                    (text, o.duration_ms, failed, verified_claim)
1657                }
1658                // Left un-resumed by `resume_undelivered` (a dirty tree
1659                // already rescues the work, or there was no session left to
1660                // resume into) — reported like the ordinary failure it is,
1661                // never as if `o.text` (the CLI's raw error JSON) were an
1662                // answer.
1663                AgentOutcome::Dropped(o) => {
1664                    let why = o
1665                        .dropped
1666                        .as_ref()
1667                        .map(|d| d.why.as_str())
1668                        .unwrap_or("the CLI ended the stream without delivering its answer");
1669                    (
1670                        String::new(),
1671                        o.duration_ms,
1672                        Some(format!("the CLI dropped the stream ({why})")),
1673                        None,
1674                    )
1675                }
1676                AgentOutcome::Quota(o) => {
1677                    self.state.quota.push(QuotaLoss {
1678                        seat: seat_key,
1679                        node: "implement".to_owned(),
1680                        at: Timestamp::now(),
1681                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1682                    });
1683                    (
1684                        String::new(),
1685                        o.duration_ms,
1686                        Some("rate limited (quota); produced no change".to_owned()),
1687                        None,
1688                    )
1689                }
1690                AgentOutcome::Failed(e) => (String::new(), 0, Some(e), None),
1691            };
1692
1693            // Rescue anything the agent edited but never committed: an
1694            // uncommitted candidate would silently be an empty one.
1695            let rescued = match git::rescue_commit(
1696                &worktree,
1697                &format!("magi: candidate {label} (uncommitted work)"),
1698            )
1699            .await
1700            {
1701                Ok(r) => {
1702                    self.state.note_withheld("implement", &r.withheld);
1703                    r.committed
1704                }
1705                Err(_) => false,
1706            };
1707            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1708                .await
1709                .unwrap_or(0);
1710            let patch = git::diff(&worktree, &base, "HEAD")
1711                .await
1712                .unwrap_or_default();
1713            let stat = git::diff_stat(&worktree, &base, "HEAD")
1714                .await
1715                .unwrap_or_default();
1716            let files = git::changed_files(&worktree, &base, "HEAD")
1717                .await
1718                .map(|f| f.len())
1719                .unwrap_or(0);
1720            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1721
1722            let c = &mut self.state.candidates[i];
1723            if !exhausted_the_fallback_chain {
1724                c.agent = agent;
1725            }
1726            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1727            c.stat = stat;
1728            c.files = files;
1729            c.commits = commits;
1730            c.duration_ms = duration;
1731            c.empty = commits == 0 || patch.trim().is_empty();
1732            // An agent that failed but still produced a committed change stays
1733            // in the running: the patch is what gets judged, not the exit code.
1734            c.failed = match failed {
1735                Some(_) if c.empty => failed,
1736                _ => None,
1737            };
1738            // Only an empty candidate can be a verified no-op: a claim next
1739            // to a real patch is not what the marker is for, and `c.failed`
1740            // being `Some` here already implies `verified_claim` was never
1741            // set (see the guard above the match that produced it).
1742            c.verified_noop = if c.empty { verified_claim } else { None };
1743            let note = match (&c.failed, c.empty, &c.verified_noop, rescued) {
1744                (Some(e), _, _, _) => format!("candidate {label}: {e}"),
1745                (None, true, Some(_), _) => {
1746                    format!("candidate {label}: no change produced (agent-verified no-op)")
1747                }
1748                (None, true, None, _) => format!("candidate {label}: no change produced"),
1749                (None, false, _, true) => {
1750                    format!(
1751                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1752                    )
1753                }
1754                (None, false, _, false) => {
1755                    format!("candidate {label}: {files} files, {commits} commits")
1756                }
1757            };
1758            self.state.event("implement", note);
1759            self.state.save()?;
1760        }
1761
1762        self.after_implement()
1763    }
1764
1765    /// Ask again, once, for work a CLI did and then failed to hand over.
1766    ///
1767    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1768    /// status with an empty response and a usage report showing output tokens,
1769    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1770    /// seven minutes and 14,267 output tokens that arrived as an empty
1771    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1772    ///
1773    /// Two conditions, and both matter:
1774    ///
1775    /// - **Only when the tree is untouched.** Often the agent has already
1776    ///   written its files and only the closing message was lost; the rescue
1777    ///   commit below picks that up and there is nothing to ask for. Re-asking
1778    ///   then would pay for a second implementation of work already on disk.
1779    /// - **Once.** A CLI that drops one stream can drop the next, and this
1780    ///   node is the most expensive in the graph.
1781    ///
1782    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1783    /// dropped reply still carried its `conversation_id`, so the seat is asked
1784    /// to finish what it was doing rather than sent the whole task again. It
1785    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1786    /// node's - for the same reason a re-ranked judge does: restating finished
1787    /// work is not the work.
1788    ///
1789    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1790    /// same way until it resets, while an abandoned conversation is still
1791    /// there to be picked up.
1792    async fn resume_undelivered(
1793        &mut self,
1794        results: &mut [(usize, SeatState, AgentOutcome)],
1795        sent: &[SeatJob],
1796        prompts: &Prompts,
1797        run_id: &str,
1798    ) {
1799        for (wi, seat, out) in results.iter_mut() {
1800            let Some(dropped) = (match &*out {
1801                AgentOutcome::Dropped(o) => o.dropped.clone(),
1802                _ => None,
1803            }) else {
1804                continue;
1805            };
1806            let Some(job) = sent.get(*wi) else { continue };
1807            // Already on disk? Then only the closing message was lost.
1808            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1809                self.state.event(
1810                    "implement",
1811                    format!(
1812                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1813                         work is in the tree",
1814                        seat.key, dropped.output_tokens, dropped.why
1815                    ),
1816                );
1817                continue;
1818            }
1819            // The re-ask only makes sense as a resume: `resume_after_drop`
1820            // says nothing about the task, trusting the seat to still hold it.
1821            // Without a session to resume — sessions disabled, or this CLI's
1822            // drop shape happened not to carry a session id — that prompt
1823            // would open a brand-new conversation with no context at all,
1824            // which is worse than leaving this as the ordinary failure it
1825            // already is.
1826            if !has_context(&job.spec, seat, job.sessions) {
1827                self.state.event(
1828                    "implement",
1829                    format!(
1830                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1831                         is no session left to resume",
1832                        seat.key, dropped.output_tokens, dropped.why
1833                    ),
1834                );
1835                continue;
1836            }
1837            self.state.event(
1838                "implement",
1839                format!(
1840                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1841                     conversation",
1842                    seat.key, dropped.output_tokens, dropped.why
1843                ),
1844            );
1845            let mut retry = job.clone();
1846            retry.seat = seat.clone();
1847            retry.prompt = prompt::resume_after_drop(&dropped.why);
1848            retry.timeout = retry_budget(job.timeout, true);
1849            retry.stem = format!("{}-resume", job.stem);
1850            let cache = self.state.config.cache_dir();
1851            let ctx = WaveCtx {
1852                carry_seats: false,
1853                run: run_id,
1854                node: "implement",
1855                prompts,
1856                cache: cache.as_deref(),
1857                round: None,
1858            };
1859            let (resumed_seat, resumed) =
1860                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1861            *seat = resumed_seat;
1862            *out = resumed;
1863        }
1864    }
1865
1866    /// Fall an implement seat through to the next untried agent in the
1867    /// implementer roster when it lost to quota — or, since the handover was
1868    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
1869    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
1870    /// quota path itself being unchanged — instead of leaving the
1871    /// seat's loss final the moment one agent's account runs dry.
1872    ///
1873    /// Solo runs (`graph.candidates = 1`, `daemon::apply_solo`'s forced shape)
1874    /// are the motivating case: `Config::resolve_roles`'s `implementers`
1875    /// truncates to the single slot rotation picked, so a solo task whose one
1876    /// implementer hits quota mid-run used to have nothing else to try. This
1877    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
1878    /// unrotated roster — which is the only place the *other* candidates in
1879    /// the machine's roster still exist once `implementers` has been cut down
1880    /// to size.
1881    ///
1882    /// Walks forward from just past the seat's own original position in the
1883    /// roster, never wrapping back to the front: a later candidate slot (say
1884    /// `beta`, the roster's second entry) must fall through to the *next*
1885    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
1886    /// almost certainly a different candidate's own agent already — and once
1887    /// the roster's tail is exhausted there is nothing left to fall through
1888    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
1889    /// whole [`AgentSpec`]: a roster with the same id named twice must not
1890    /// let this retry that id forever. The loop keeps falling through until
1891    /// an attempt lands something other than `Quota` or the roster's tail
1892    /// runs out of untried ids, at which point the seat is left exactly as
1893    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
1894    /// `QuotaLoss` recorded, the candidate failed/empty.
1895    ///
1896    /// `sent` is taken mutably and updated with the fallback agent's spec:
1897    /// `resume_unconfirmed_commands`, which runs after this and also reads
1898    /// `sent`, must see whichever agent actually ended up answering the seat
1899    /// — reading the stale, original spec there would check session
1900    /// eligibility against the wrong CLI and could hand a fallback agent's
1901    /// session id to the agent that just lost the seat to quota.
1902    ///
1903    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
1904    /// — `self.seat` only reuses state when the agent id is unchanged, so
1905    /// handing it a different id already gets this for free. Reusing the old
1906    /// seat would resume a different CLI's session as if it were a
1907    /// continuation of this one.
1908    ///
1909    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
1910    /// a quota loss cuts an agent off mid-turn, so anything already in the
1911    /// tree is unfinished work, not a completed candidate a re-ask would pay
1912    /// for twice. A dirty tree is rescued into a commit first (the same
1913    /// neutral-identity rescue `implement`'s own outcome loop gives every
1914    /// candidate) so the next agent starts clean.
1915    ///
1916    /// The new agent gets the implementer's full prompt and full
1917    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
1918    /// it has no session and no context, and is implementing the task from
1919    /// nothing, unlike a resumed drop which is only restating work already
1920    /// done.
1921    ///
1922    /// Every intermediate `Quota` this loop absorbs is folded into a plain
1923    /// `implement` event, never into `self.state.quota` — that is what
1924    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
1925    /// go unspent, and a seat that ultimately recovered on its second or
1926    /// third agent is not the stalled panel that check exists to catch. Only
1927    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
1928    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
1929    /// outcome loop already has — this helper never pushes to it itself.
1930    async fn resume_seat_handovers(
1931        &mut self,
1932        results: &mut [(usize, SeatState, AgentOutcome)],
1933        sent: &mut [SeatJob],
1934        prompts: &Prompts,
1935        run_id: &str,
1936    ) {
1937        let instruction = seeded_instruction(&self.state);
1938        let language = self.state.config.graph.language.clone();
1939        let brief = self
1940            .state
1941            .advice
1942            .as_ref()
1943            .and_then(|a| a.synthesis.as_deref())
1944            .map(str::to_owned);
1945        let attachments = self.state.attachments.clone();
1946        for (wi, seat, out) in results.iter_mut() {
1947            let Some(job) = sent.get_mut(*wi) else {
1948                continue;
1949            };
1950            // Where the seat's own original agent sits in the roster — the
1951            // fallback walk starts just past here, never at the front, so a
1952            // later candidate slot's quota loss does not fall back onto an
1953            // earlier slot's own agent.
1954            let start = self
1955                .roles
1956                .implementer_roster
1957                .iter()
1958                .position(|s| s.id == job.spec.id)
1959                .unwrap_or(0);
1960            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
1961            let mut fallback_attempt = 0usize;
1962            let mut prev: Option<FailClass> = None;
1963            while let Some(cur) = FailClass::of(&*out) {
1964                if !should_hand_over(prev.as_ref(), &cur) {
1965                    break;
1966                }
1967                let Some(next) =
1968                    next_untried_in_roster(&self.roles.implementer_roster, start, &tried).cloned()
1969                else {
1970                    break;
1971                };
1972                tried.insert(next.id.clone());
1973                fallback_attempt += 1;
1974
1975                if let Ok(r) = git::rescue_commit(
1976                    &job.cwd,
1977                    &format!(
1978                        "magi: candidate {} (uncommitted work before {} fallback)",
1979                        seat.key,
1980                        if cur == FailClass::Quota {
1981                            "quota"
1982                        } else {
1983                            "handover"
1984                        }
1985                    ),
1986                )
1987                .await
1988                {
1989                    self.state.note_withheld("implement", &r.withheld);
1990                }
1991
1992                record_handover(
1993                    &mut self.state,
1994                    "implement",
1995                    &seat.key,
1996                    &seat.agent,
1997                    &next.id,
1998                    &cur,
1999                    &fail_reason(&*out),
2000                );
2001                prev = Some(cur.clone());
2002
2003                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
2004                self.state.seats.insert(seat.key.clone(), new_seat.clone());
2005                // Kept in sync on `sent` itself, not just the local retry: a
2006                // later helper (`resume_unconfirmed_commands`) reads `sent`
2007                // after this one returns and must see whichever agent is now
2008                // occupying the seat, not the one that just quota'd out —
2009                // otherwise it would judge session/continuation eligibility
2010                // by the wrong CLI and could resend a fallback's session id
2011                // to the agent that lost it the seat in the first place.
2012                job.spec = next.clone();
2013                let mut retry = job.clone();
2014                retry.seat = new_seat;
2015                retry.prompt = prompt::implement(
2016                    &instruction,
2017                    &job.cwd.to_string_lossy(),
2018                    &language,
2019                    brief.as_deref(),
2020                    &attachments,
2021                );
2022                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
2023                let cache = self.state.config.cache_dir();
2024                let ctx = WaveCtx {
2025                    carry_seats: false,
2026                    run: run_id,
2027                    node: "implement",
2028                    prompts,
2029                    cache: cache.as_deref(),
2030                    round: None,
2031                };
2032                let (fallback_seat, fallback_out) = run_one(
2033                    retry,
2034                    Arc::clone(&self.sem),
2035                    &ctx,
2036                    &mut self.state,
2037                    fallback_attempt,
2038                )
2039                .await;
2040                *seat = fallback_seat;
2041                *out = fallback_out;
2042            }
2043        }
2044    }
2045
2046    /// Ask an implement seat's own CLI to confirm what it started, once, when
2047    /// its reply reported a command whose completion status it never
2048    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
2049    /// that does and does not mean.
2050    ///
2051    /// The completion contract this task asks for, extended to `implement`
2052    /// with the same signal `continue_fix_report` reads for the fixer,
2053    /// rather than a keyword search over the reply or a hard requirement on
2054    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
2055    /// a candidate's CLI turn ended cleanly while a test run it had started
2056    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
2057    /// named in it at all is untouched by this: `commands` is empty, so
2058    /// there is nothing to be unconfirmed.
2059    ///
2060    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
2061    /// this is not about recovering edits that might already be on disk, it
2062    /// is about a result the seat itself never vouched for, which resuming
2063    /// asks for regardless of what the tree already holds. Bounded to one
2064    /// attempt for the same reason `resume_undelivered` is — this is the
2065    /// most expensive node in the graph — and a seat that still cannot
2066    /// confirm on that attempt is left as whatever its (possibly still
2067    /// unconfirmed) reply says; this does not invent a new "failed" reason
2068    /// for a candidate that otherwise produced a real, committed change.
2069    async fn resume_unconfirmed_commands(
2070        &mut self,
2071        results: &mut [(usize, SeatState, AgentOutcome)],
2072        sent: &[SeatJob],
2073        prompts: &Prompts,
2074        run_id: &str,
2075    ) {
2076        for (wi, seat, out) in results.iter_mut() {
2077            let AgentOutcome::Ok(o) = &*out else {
2078                continue;
2079            };
2080            if !has_unconfirmed_command(&o.commands) {
2081                continue;
2082            }
2083            let Some(job) = sent.get(*wi) else { continue };
2084            if !has_context(&job.spec, seat, job.sessions) {
2085                self.state.event(
2086                    "implement",
2087                    format!(
2088                        "{}: the reply named a command whose own CLI never confirmed the exit \
2089                         status of, but there is no session left to resume",
2090                        seat.key
2091                    ),
2092                );
2093                continue;
2094            }
2095            self.state.event(
2096                "implement",
2097                format!(
2098                    "{}: the reply named a command whose own CLI never confirmed the exit \
2099                     status of; resuming the conversation",
2100                    seat.key
2101                ),
2102            );
2103            let mut retry = job.clone();
2104            retry.seat = seat.clone();
2105            retry.prompt = prompt::resume_incomplete(
2106                "a command in your last reply had no confirmed exit status",
2107            );
2108            retry.timeout = retry_budget(job.timeout, true);
2109            retry.stem = format!("{}-confirm", job.stem);
2110            let cache = self.state.config.cache_dir();
2111            let ctx = WaveCtx {
2112                carry_seats: false,
2113                run: run_id,
2114                node: "implement",
2115                prompts,
2116                cache: cache.as_deref(),
2117                round: None,
2118            };
2119            let (resumed_seat, resumed) =
2120                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
2121            *seat = resumed_seat;
2122            *out = resumed;
2123        }
2124    }
2125
2126    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
2127    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
2128    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
2129    /// that motivated this.
2130    ///
2131    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
2132    /// own nudge loop already covers for judge/review/vote seats, and not a
2133    /// dropped stream, which [`Runner::resume_undelivered`] covers for
2134    /// implement seats: here the CLI turn genuinely finished while the node's
2135    /// own work — the fixer's account of what it did — had not. Gated purely
2136    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
2137    /// reply, never on any wording in it, so a fixer whose valid, first-try
2138    /// `FixReport` happens to mention having waited on a background test is
2139    /// never resumed — the `Ok(report)` branch at the call site returns
2140    /// before this is ever invoked.
2141    ///
2142    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
2143    /// attempt ([`retry_budget`]), nothing attempted once the session is
2144    /// gone, and a quota hit ends the loop immediately rather than retrying a
2145    /// rate limit that fails the same way again.
2146    async fn continue_fix_report(
2147        &mut self,
2148        mut seat: SeatState,
2149        parse_err: String,
2150        job: &SeatJob,
2151        prompts: &Prompts,
2152        run_id: &str,
2153        round: usize,
2154    ) -> (
2155        SeatState,
2156        Option<FixReport>,
2157        Option<String>,
2158        ContinuationRecord,
2159    ) {
2160        let mut last_err = parse_err;
2161        let mut cumulative_wait_ms = 0u64;
2162        let mut attempts = 0usize;
2163        loop {
2164            if !has_context(&job.spec, &seat, job.sessions) {
2165                self.state.event(
2166                    "fix",
2167                    format!(
2168                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
2169                         session left to resume into"
2170                    ),
2171                );
2172                let outcome = if attempts == 0 {
2173                    ContinuationOutcome::NoSession
2174                } else {
2175                    ContinuationOutcome::Exhausted
2176                };
2177                return (
2178                    seat,
2179                    None,
2180                    Some(format!("unparsable fix report: {last_err}")),
2181                    ContinuationRecord {
2182                        attempts,
2183                        cumulative_wait_ms,
2184                        outcome,
2185                    },
2186                );
2187            }
2188            if attempts >= MAX_FIX_CONTINUATIONS {
2189                self.state.event(
2190                    "fix",
2191                    format!(
2192                        "round {round}: fixer's reply still had no adoption report after \
2193                         {attempts} continuation(s) ({last_err}); giving up"
2194                    ),
2195                );
2196                return (
2197                    seat,
2198                    None,
2199                    Some(format!(
2200                        "unparsable fix report after {attempts} continuation(s): {last_err}"
2201                    )),
2202                    ContinuationRecord {
2203                        attempts,
2204                        cumulative_wait_ms,
2205                        outcome: ContinuationOutcome::Exhausted,
2206                    },
2207                );
2208            }
2209            attempts += 1;
2210            self.state.event(
2211                "fix",
2212                format!(
2213                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
2214                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
2215                ),
2216            );
2217            let mut retry = job.clone();
2218            retry.seat = seat.clone();
2219            retry.prompt = prompt::resume_incomplete(&last_err);
2220            retry.timeout = retry_budget(job.timeout, true);
2221            retry.stem = format!("{}-continue{attempts}", job.stem);
2222            let cache = self.state.config.cache_dir();
2223            let ctx = WaveCtx {
2224                carry_seats: false,
2225                run: run_id,
2226                node: "fix",
2227                prompts,
2228                cache: cache.as_deref(),
2229                round: Some(round),
2230            };
2231            let (resumed_seat, resumed_out) = run_one(
2232                retry,
2233                Arc::clone(&self.sem),
2234                &ctx,
2235                &mut self.state,
2236                attempts,
2237            )
2238            .await;
2239            seat = resumed_seat;
2240            match resumed_out {
2241                AgentOutcome::Ok(o) => {
2242                    cumulative_wait_ms += o.duration_ms;
2243                    match verdict::extract_json::<FixReport>(&o.text) {
2244                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
2245                            self.state.event(
2246                                "fix",
2247                                format!(
2248                                    "round {round}: fixer's adoption report recovered after \
2249                                     {attempts} continuation(s)"
2250                                ),
2251                            );
2252                            return (
2253                                seat,
2254                                Some(report),
2255                                None,
2256                                ContinuationRecord {
2257                                    attempts,
2258                                    cumulative_wait_ms,
2259                                    outcome: ContinuationOutcome::Resumed,
2260                                },
2261                            );
2262                        }
2263                        // The report parsed, but this same reply's own
2264                        // CommandEvidence — the identical record `state.jobs`
2265                        // renders — names a command whose CLI never
2266                        // confirmed an exit status. Read together, that is
2267                        // not a resolved answer: keep nudging rather than
2268                        // accept a report standing next to a command the
2269                        // seat's own CLI cannot vouch for.
2270                        Ok(_) => {
2271                            last_err = "the reply parsed, but it reported a command whose own CLI \
2272                                 never confirmed an exit status"
2273                                .to_owned();
2274                        }
2275                        Err(e) => last_err = e.to_string(),
2276                    }
2277                }
2278                AgentOutcome::Quota(o) => {
2279                    cumulative_wait_ms += o.duration_ms;
2280                    self.state.quota.push(QuotaLoss {
2281                        seat: seat.key.clone(),
2282                        node: "fix".to_owned(),
2283                        at: Timestamp::now(),
2284                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2285                    });
2286                    self.state.event(
2287                        "fix",
2288                        format!(
2289                            "round {round}: continuation rate limited (quota); not retrying now"
2290                        ),
2291                    );
2292                    return (
2293                        seat,
2294                        None,
2295                        Some("rate limited (quota) while recovering the fix report".to_owned()),
2296                        ContinuationRecord {
2297                            attempts,
2298                            cumulative_wait_ms,
2299                            outcome: ContinuationOutcome::QuotaLost,
2300                        },
2301                    );
2302                }
2303                AgentOutcome::Dropped(o) => {
2304                    cumulative_wait_ms += o.duration_ms;
2305                    let why = o
2306                        .dropped
2307                        .as_ref()
2308                        .map(|d| d.why.as_str())
2309                        .unwrap_or("the CLI ended the stream without delivering its answer");
2310                    last_err = format!("the CLI dropped the stream ({why})");
2311                }
2312                AgentOutcome::Failed(e) => last_err = e,
2313            }
2314        }
2315    }
2316
2317    fn after_implement(&mut self) -> Result<()> {
2318        // Scan every candidate patch once the set is complete.
2319        if self.state.leaks.is_empty() {
2320            let cfg = self.state.config.blind.clone();
2321            let mut leaks = Vec::new();
2322            for c in &self.state.candidates {
2323                let Some(patch) =
2324                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2325                else {
2326                    continue;
2327                };
2328                leaks.extend(blind::scan(
2329                    &format!("candidate {} patch", c.label),
2330                    &patch,
2331                    &cfg.vendor_tokens,
2332                ));
2333            }
2334            if !leaks.is_empty() {
2335                let summary = leaks
2336                    .iter()
2337                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
2338                    .collect::<Vec<_>>()
2339                    .join(", ");
2340                match cfg.on_leak {
2341                    LeakPolicy::Fail => {
2342                        self.state.status = RunStatus::Failed;
2343                        self.state
2344                            .event("blind", format!("vendor text in a patch: {summary}"));
2345                        self.state.leaks = leaks;
2346                        self.state.save()?;
2347                        self.settle_questions();
2348                        bail!(
2349                            "blind.on_leak = \"fail\" and vendor text reached a \
2350                             judged patch: {summary}"
2351                        );
2352                    }
2353                    LeakPolicy::Redact => self.state.event(
2354                        "blind",
2355                        format!("redacting vendor text for judging: {summary}"),
2356                    ),
2357                    LeakPolicy::Warn => self.state.event(
2358                        "blind",
2359                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
2360                    ),
2361                }
2362                self.state.leaks = leaks;
2363            }
2364        }
2365
2366        if self.state.viable().is_empty() {
2367            if self.state.all_candidates_verified_noop() {
2368                // Every candidate agreed, with evidence the adoption guard
2369                // accepted, that nothing belongs in this worktree. That is
2370                // not the same fact as a candidate that simply failed to
2371                // write anything, and settling it as an ordinary `Failed`
2372                // (see `SCHEMA`'s doc for schema 10) is what let two of
2373                // task 391f's attempts burn a retry each re-discovering the
2374                // same already-landed fix. Terminal either way, so `judge`
2375                // must never run over an empty candidate set — unlike the
2376                // `Failed` branch below this returns `Ok`, not an error:
2377                // nothing here failed.
2378                self.state.status = RunStatus::VerifiedNoop;
2379                self.state.save()?;
2380                self.settle_questions();
2381                return Ok(());
2382            }
2383            self.state.status = RunStatus::Failed;
2384            self.state.save()?;
2385            self.settle_questions();
2386            bail!("no candidate produced a change; nothing to judge");
2387        }
2388        self.state.status = RunStatus::Judging;
2389        self.state.save()?;
2390        Ok(())
2391    }
2392
2393    // --------------------------------------------------------------- judge
2394
2395    async fn judge(&mut self) -> Result<()> {
2396        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2397        // agent files with `magi task add` name the run that paid for it. The
2398        // prompt overlay is cloned alongside it because the waves borrow it
2399        // while `self` is mutably borrowed by the node's own bookkeeping.
2400        let run_id = self.state.id.clone();
2401        let prompts = self.state.config.prompts.clone();
2402        if !self.state.judgements.is_empty() || self.state.judge_skipped {
2403            return Ok(());
2404        }
2405        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2406        if viable.len() == 1 {
2407            // Recorded so this is a one-time event: `judgements` stays empty
2408            // either way, which without this flag is indistinguishable from
2409            // "not yet judged" on the next reentry — and status is left
2410            // untouched, so a later node's conclusion (e.g. `Blocked` after
2411            // the review budget ran out) survives a resume instead of being
2412            // clobbered back to `Judging` by this node running again.
2413            self.state.judge_skipped = true;
2414            self.state.event(
2415                "judge",
2416                format!(
2417                    "only candidate {} produced a change; judging skipped",
2418                    viable[0].label
2419                ),
2420            );
2421            self.state.save()?;
2422            return Ok(());
2423        }
2424        self.state.status = RunStatus::Judging;
2425
2426        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2427        let language = self.state.config.graph.language.clone();
2428        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2429        let sessions = self.state.config.graph.sessions;
2430        let artifacts = agent::artifacts_dir(&self.state.dir());
2431        let root = self.state.worktree_root();
2432        let base_short = short(&self.state.base_commit);
2433
2434        let mut jobs = Vec::new();
2435        let mut orders = Vec::new();
2436        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2437            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2438            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2439            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
2440            let seat_key = format!("judge-{}", j + 1);
2441            let seat = self.seat(&seat_key, &spec.id);
2442            jobs.push(SeatJob {
2443                prompt: prompt::judge(
2444                    &self.state.instruction,
2445                    &views,
2446                    self.roles.judges.len(),
2447                    &base_short,
2448                    &language,
2449                ),
2450                spec,
2451                seat,
2452                cwd: root.join(format!("judge-{}", j + 1)),
2453                timeout,
2454                allow_write: false,
2455                sessions,
2456                artifacts: artifacts.clone(),
2457                stem: format!("judge-{}", j + 1),
2458                handover: None,
2459            });
2460        }
2461
2462        self.state.event(
2463            "judge",
2464            format!(
2465                "{} judges ranking {} candidates blind",
2466                jobs.len(),
2467                viable.len()
2468            ),
2469        );
2470        let labels_for_check = labels.clone();
2471        let mut quota_losses = Vec::new();
2472        let cache = self.state.config.cache_dir();
2473        let ctx = WaveCtx {
2474            carry_seats: false,
2475            run: &run_id,
2476            node: "judge",
2477            prompts: &prompts,
2478            cache: cache.as_deref(),
2479            round: None,
2480        };
2481        let results = ask_json_wave::<Ranking>(
2482            jobs,
2483            Arc::clone(&self.sem),
2484            self.state.config.graph.retries,
2485            &self.roles.judge_roster,
2486            &ctx,
2487            &mut quota_losses,
2488            &mut self.state,
2489            &move |r: &Ranking| r.validate(&labels_for_check),
2490        )
2491        .await;
2492        self.state.quota.extend(quota_losses);
2493
2494        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
2495            let agent_id = seat.agent.clone();
2496            self.state.seats.insert(seat.key.clone(), seat);
2497            let mut record = Judgement {
2498                judge: j + 1,
2499                seat: format!("judge-{}", j + 1),
2500                agent: agent_id,
2501                ranking: Vec::new(),
2502                reasons: BTreeMap::new(),
2503                confidence: None,
2504                order: orders[j].clone(),
2505                failed: None,
2506                duration_ms: 0,
2507            };
2508            match res {
2509                Ok((ranking, out)) => {
2510                    record.ranking = ranking.normalized();
2511                    record.reasons = ranking.reasons;
2512                    record.confidence = ranking.confidence;
2513                    record.duration_ms = out.duration_ms;
2514                    self.state.event(
2515                        "judge",
2516                        format!(
2517                            "judge {} ranked {}",
2518                            j + 1,
2519                            record.ranking.iter().collect::<String>()
2520                        ),
2521                    );
2522                }
2523                Err(e) => {
2524                    record.failed = Some(e.to_string());
2525                    self.state
2526                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
2527                }
2528            }
2529            self.state.judgements.push(record);
2530            self.state.save()?;
2531        }
2532        Ok(())
2533    }
2534
2535    // ---------------------------------------------------------- deliberate
2536
2537    async fn deliberate(&mut self) -> Result<()> {
2538        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2539        // agent files with `magi task add` name the run that paid for it. The
2540        // prompt overlay is cloned alongside it because the waves borrow it
2541        // while `self` is mutably borrowed by the node's own bookkeeping.
2542        let run_id = self.state.id.clone();
2543        let prompts = self.state.config.prompts.clone();
2544        if !self.state.deliberation.is_empty() {
2545            return Ok(());
2546        }
2547        let tops: Vec<char> = self
2548            .state
2549            .judgements
2550            .iter()
2551            .filter_map(|j| j.ranking.first().copied())
2552            .collect();
2553        let rounds = self.state.config.graph.deliberate_rounds;
2554        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
2555            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
2556                self.state.event(
2557                    "deliberate",
2558                    format!("judges agreed on {} outright; no deliberation", tops[0]),
2559                );
2560            }
2561            self.state.status = RunStatus::Voting;
2562            self.state.save()?;
2563            return Ok(());
2564        }
2565
2566        self.state.status = RunStatus::Deliberating;
2567        self.state.event(
2568            "deliberate",
2569            format!(
2570                "split: first choices were {} — opening {rounds} round(s)",
2571                tops.iter().collect::<String>()
2572            ),
2573        );
2574
2575        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2576        let language = self.state.config.graph.language.clone();
2577        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2578        let sessions = self.state.config.graph.sessions;
2579        let artifacts = agent::artifacts_dir(&self.state.dir());
2580        let root = self.state.worktree_root();
2581        let base_short = short(&self.state.base_commit);
2582
2583        // Judges argue in sequence so that a turn can answer the one before it;
2584        // that is the difference between deliberation and three parallel
2585        // monologues.
2586        for round in 1..=rounds {
2587            let mut turns: Vec<DeliberationTurn> = Vec::new();
2588            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2589                if self.state.judgements[j].failed.is_some() {
2590                    continue;
2591                }
2592                let seat_key = format!("judge-{}", j + 1);
2593                let spec = self.occupant(&seat_key, spec);
2594                let mut seat = self.seat(&seat_key, &spec.id);
2595                let transcript = self.transcript(&turns, j);
2596                let build = |context: Option<&str>| {
2597                    prompt::deliberate(
2598                        &self.state.instruction,
2599                        context,
2600                        &transcript,
2601                        round,
2602                        rounds,
2603                        &language,
2604                    )
2605                };
2606                let block = self.candidate_block(&viable, &base_short);
2607                let full = build(Some(&block));
2608                let text = if has_context(&spec, &seat, sessions) {
2609                    build(None)
2610                } else {
2611                    full.clone()
2612                };
2613                let job = SeatJob {
2614                    spec,
2615                    seat: seat.clone(),
2616                    prompt: text,
2617                    cwd: root.join(format!("judge-{}", j + 1)),
2618                    timeout,
2619                    allow_write: false,
2620                    sessions,
2621                    artifacts: artifacts.clone(),
2622                    stem: format!("delib-{round}-judge-{}", j + 1),
2623                    handover: Some(full),
2624                };
2625                let cache = self.state.config.cache_dir();
2626                let ctx = WaveCtx {
2627                    carry_seats: false,
2628                    run: &run_id,
2629                    node: "deliberate",
2630                    prompts: &prompts,
2631                    cache: cache.as_deref(),
2632                    round: None,
2633                };
2634                // A turn is never nudged (`retries` 0): a failed seat is
2635                // handed to the next roster agent, which gets the full
2636                // context. An empty answer is a turn, not a failure.
2637                let mut losses = Vec::new();
2638                let mut results = ask_wave_with::<String>(
2639                    vec![job],
2640                    Arc::clone(&self.sem),
2641                    0,
2642                    &self.roles.judge_roster,
2643                    &ctx,
2644                    &mut losses,
2645                    &mut self.state,
2646                    &|text: &str| {
2647                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
2648                    },
2649                )
2650                .await;
2651                self.state.quota.extend(losses);
2652                let (updated, res, _) = results.pop().expect("one job in, one result out");
2653                seat = updated;
2654                let agent_id = seat.agent.clone();
2655                self.state.seats.insert(seat.key.clone(), seat);
2656                let body = match res {
2657                    Ok((body, _)) => body,
2658                    // Skip the seat; a CLI's raw error JSON is never read as
2659                    // this judge's position.
2660                    Err(e) => {
2661                        self.state
2662                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
2663                        continue;
2664                    }
2665                };
2666                let tentative = verdict::extract_json::<Position>(&body)
2667                    .ok()
2668                    .and_then(|p| p.tentative)
2669                    .and_then(|s| s.trim().chars().next())
2670                    .map(|c| c.to_ascii_uppercase());
2671                self.state.event(
2672                    "deliberate",
2673                    format!(
2674                        "round {round}: judge {} now favours {}",
2675                        j + 1,
2676                        tentative.map_or("—".to_owned(), |c| c.to_string())
2677                    ),
2678                );
2679                turns.push(DeliberationTurn {
2680                    judge: j + 1,
2681                    agent: agent_id,
2682                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2683                    tentative,
2684                });
2685            }
2686            self.state
2687                .deliberation
2688                .push(DeliberationRound { round, turns });
2689            self.state.save()?;
2690        }
2691
2692        self.state.status = RunStatus::Voting;
2693        self.state.save()?;
2694        Ok(())
2695    }
2696
2697    // ---------------------------------------------------------------- vote
2698
2699    async fn vote(&mut self) -> Result<()> {
2700        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2701        // agent files with `magi task add` name the run that paid for it. The
2702        // prompt overlay is cloned alongside it because the waves borrow it
2703        // while `self` is mutably borrowed by the node's own bookkeeping.
2704        let run_id = self.state.id.clone();
2705        let prompts = self.state.config.prompts.clone();
2706        if !self.state.votes.is_empty() {
2707            return Ok(());
2708        }
2709        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2710        if viable.len() == 1 {
2711            return Ok(());
2712        }
2713        self.state.status = RunStatus::Voting;
2714
2715        let language = self.state.config.graph.language.clone();
2716        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2717        let sessions = self.state.config.graph.sessions;
2718        let artifacts = agent::artifacts_dir(&self.state.dir());
2719        let root = self.state.worktree_root();
2720        let base_short = short(&self.state.base_commit);
2721        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2722
2723        let mut jobs = Vec::new();
2724        let mut seats_at = Vec::new();
2725        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2726            if self
2727                .state
2728                .judgements
2729                .get(j)
2730                .is_some_and(|r| r.failed.is_some())
2731            {
2732                continue;
2733            }
2734            let seat_key = format!("judge-{}", j + 1);
2735            let spec = self.occupant(&seat_key, spec);
2736            let seat = self.seat(&seat_key, &spec.id);
2737            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
2738            let text = if has_context(&spec, &seat, sessions) {
2739                prompt::final_vote(&viable, &language)
2740            } else {
2741                full.clone()
2742            };
2743            jobs.push(SeatJob {
2744                spec,
2745                seat,
2746                prompt: text,
2747                cwd: root.join(format!("judge-{}", j + 1)),
2748                timeout,
2749                allow_write: false,
2750                sessions,
2751                artifacts: artifacts.clone(),
2752                stem: format!("vote-judge-{}", j + 1),
2753                handover: Some(full),
2754            });
2755            seats_at.push(j);
2756        }
2757
2758        self.state.event(
2759            "vote",
2760            format!(
2761                "collecting {} final votes one by one, privately",
2762                jobs.len()
2763            ),
2764        );
2765        let allowed = viable.clone();
2766        let mut quota_losses = Vec::new();
2767        let cache = self.state.config.cache_dir();
2768        let ctx = WaveCtx {
2769            carry_seats: false,
2770            run: &run_id,
2771            node: "vote",
2772            prompts: &prompts,
2773            cache: cache.as_deref(),
2774            round: None,
2775        };
2776        let results = ask_json_wave::<FinalVote>(
2777            jobs,
2778            Arc::clone(&self.sem),
2779            self.state.config.graph.retries,
2780            &self.roles.judge_roster,
2781            &ctx,
2782            &mut quota_losses,
2783            &mut self.state,
2784            &move |v: &FinalVote| match v.label() {
2785                Some(c) if allowed.contains(&c) => Ok(()),
2786                other => bail!("vote {other:?} is not one of {allowed:?}"),
2787            },
2788        )
2789        .await;
2790        self.state.quota.extend(quota_losses);
2791
2792        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
2793            let agent_id = seat.agent.clone();
2794            self.state.seats.insert(seat.key.clone(), seat);
2795            let initial = self
2796                .state
2797                .judgements
2798                .get(j)
2799                .and_then(|r| r.ranking.first().copied());
2800            let mut record = VoteRecord {
2801                judge: j + 1,
2802                agent: agent_id,
2803                vote: None,
2804                reason: String::new(),
2805                changed: false,
2806            };
2807            match res {
2808                Ok((v, _)) => {
2809                    record.vote = v.label();
2810                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2811                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2812                    self.state.event(
2813                        "vote",
2814                        format!(
2815                            "judge {} voted {}{}",
2816                            j + 1,
2817                            record.vote.unwrap_or('?'),
2818                            if record.changed { " (changed)" } else { "" }
2819                        ),
2820                    );
2821                }
2822                Err(e) => {
2823                    self.state
2824                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2825                }
2826            }
2827            self.state.votes.push(record);
2828            self.state.save()?;
2829        }
2830        Ok(())
2831    }
2832
2833    // --------------------------------------------------------------- tally
2834
2835    fn tally(&mut self) -> Result<()> {
2836        if self.state.tally.is_some() {
2837            return Ok(());
2838        }
2839        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2840        let tops: Vec<char> = self
2841            .state
2842            .judgements
2843            .iter()
2844            .filter_map(|j| j.ranking.first().copied())
2845            .collect();
2846        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2847
2848        // A judge whose private vote failed still counted once, in the initial
2849        // ranking; using it beats discarding a whole seat.
2850        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2851        let mut cast: Vec<char> = Vec::new();
2852        for (i, j) in self.state.judgements.iter().enumerate() {
2853            let vote = self
2854                .state
2855                .votes
2856                .iter()
2857                .find(|v| v.judge == i + 1)
2858                .and_then(|v| v.vote)
2859                .or_else(|| j.ranking.first().copied());
2860            if let Some(v) = vote {
2861                *first_choice.entry(v).or_insert(0) += 1;
2862                cast.push(v);
2863            }
2864        }
2865
2866        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2867        for j in &self.state.judgements {
2868            let n = j.ranking.len();
2869            for (pos, label) in j.ranking.iter().enumerate() {
2870                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2871            }
2872        }
2873
2874        let best = first_choice.values().copied().max().unwrap_or(0);
2875        let mut leaders: Vec<char> = first_choice
2876            .iter()
2877            .filter(|(_, v)| **v == best)
2878            .map(|(k, _)| *k)
2879            .collect();
2880        let mut tie_break = None;
2881        if leaders.len() > 1 {
2882            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2883            let borda_leaders: Vec<char> = leaders
2884                .iter()
2885                .copied()
2886                .filter(|l| borda[l] == top_borda)
2887                .collect();
2888            tie_break = Some(if borda_leaders.len() == 1 {
2889                format!(
2890                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2891                    leaders.len()
2892                )
2893            } else {
2894                format!(
2895                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2896                    leaders.len()
2897                )
2898            });
2899            leaders = borda_leaders;
2900            leaders.sort_unstable();
2901        }
2902        let winner = *leaders
2903            .first()
2904            .or(viable.first())
2905            .context("no candidate to declare a winner from")?;
2906
2907        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2908        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2909        let deliberated = !self.state.deliberation.is_empty();
2910
2911        // Whose verdict is this? A rate-limited seat is absent even if it
2912        // ranked before the limit hit, so presence is measured against the
2913        // recorded losses, not just "did a ranking ever appear".
2914        let quota_seats: std::collections::BTreeSet<&str> =
2915            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2916        let mut present = 0usize;
2917        for (i, j) in self.state.judgements.iter().enumerate() {
2918            if quota_seats.contains(j.seat.as_str()) {
2919                continue;
2920            }
2921            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2922            let voted = self
2923                .state
2924                .votes
2925                .iter()
2926                .any(|v| v.judge == i + 1 && v.vote.is_some());
2927            if ranked || voted {
2928                present += 1;
2929            }
2930        }
2931        // Strict majority of the configured panel. A bare majority is real
2932        // signal we can act on, while a minority verdict must never stand in
2933        // for a healthy one. A one-candidate run needs no panel at all, and
2934        // `judges` stays `0` rather than the roster size a panel that never
2935        // sat would otherwise be credited with.
2936        let needs_quorum = viable.len() > 1;
2937        let judges_total = if needs_quorum {
2938            self.roles.judges.len()
2939        } else {
2940            0
2941        };
2942        let quorum = if needs_quorum {
2943            judges_total / 2 + 1
2944        } else {
2945            0
2946        };
2947        let met_quorum = !needs_quorum || present >= quorum;
2948        let uncontested = (!needs_quorum).then(|| {
2949            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2950        });
2951
2952        self.state.event(
2953            "tally",
2954            match &uncontested {
2955                Some(reason) => format!("winner {winner} — {reason}"),
2956                None => format!(
2957                    "winner {winner} — votes {} | initial {} | {} changed | \
2958                     {present}/{judges_total} judges{}",
2959                    first_choice
2960                        .iter()
2961                        .map(|(k, v)| format!("{k}:{v}"))
2962                        .collect::<Vec<_>>()
2963                        .join(" "),
2964                    if unanimous_initial {
2965                        "unanimous"
2966                    } else {
2967                        "split"
2968                    },
2969                    changed_votes,
2970                    if met_quorum {
2971                        String::new()
2972                    } else {
2973                        format!(" — below quorum ({quorum} required)")
2974                    },
2975                ),
2976            },
2977        );
2978        if !met_quorum {
2979            self.state.event(
2980                "stall",
2981                format!(
2982                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2983                     the run stops here, resumable"
2984                ),
2985            );
2986        }
2987        self.state.tally = Some(Tally {
2988            first_choice,
2989            borda,
2990            winner,
2991            rankings: tops.len(),
2992            unanimous_initial,
2993            deliberated,
2994            changed_votes,
2995            unanimous_final,
2996            tie_break,
2997            judges: judges_total,
2998            present,
2999            quorum,
3000            met_quorum,
3001            uncontested,
3002        });
3003        self.state.status = if met_quorum {
3004            RunStatus::Reviewing
3005        } else {
3006            RunStatus::Stalled
3007        };
3008        self.state.save()?;
3009        Ok(())
3010    }
3011
3012    // ------------------------------------------------------------- recover
3013
3014    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
3015    /// resumed toward completion once the transient cause clears.
3016    ///
3017    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
3018    /// it toward the quorum, which is exactly the set of seats whose absence
3019    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
3020    /// not depend on which node happened to hit the limit), or an ordinary
3021    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
3022    /// seat is never disturbed.
3023    ///
3024    /// A seat that now answers with a usable ranking is "recovered": its
3025    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
3026    /// `tally` counts it present again), and its vote re-collected. A seat that
3027    /// still fails keeps its loss and stays absent.
3028    ///
3029    /// Returns `true` when the re-tally restores the quorum (the run may proceed
3030    /// to review/gate/merge), `false` when it is still below quorum (the run
3031    /// stays `Stalled`, still resumable for a later retry).
3032    #[allow(clippy::too_many_lines)]
3033    async fn recover_stall(&mut self) -> Result<bool> {
3034        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3035        // agent files with `magi task add` name the run that paid for it. The
3036        // prompt overlay is cloned alongside it because the waves borrow it
3037        // while `self` is mutably borrowed by the node's own bookkeeping.
3038        let run_id = self.state.id.clone();
3039        let prompts = self.state.config.prompts.clone();
3040        // Absent seats = quota-lost at any node, or failed outright. Mirroring
3041        // `tally`'s presence test (rather than the old quota-judge/vote filter)
3042        // is what keeps a non-quota collapse — or a quota loss recorded at the
3043        // deliberate node — from being a permanent dead-end on `--resume`.
3044        let quota_seats: BTreeSet<&str> =
3045            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
3046        let absent: Vec<String> = self
3047            .state
3048            .judgements
3049            .iter()
3050            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
3051            .map(|j| j.seat.clone())
3052            .collect();
3053        if absent.is_empty() {
3054            return Ok(false);
3055        }
3056        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3057        if viable.len() <= 1 {
3058            return Ok(false);
3059        }
3060        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3061        let language = self.state.config.graph.language.clone();
3062        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3063        let sessions = self.state.config.graph.sessions;
3064        let artifacts = agent::artifacts_dir(&self.state.dir());
3065        let root = self.state.worktree_root();
3066        let base_short = short(&self.state.base_commit);
3067        let candidates: Vec<Candidate> = viable.clone();
3068
3069        // Map each absent seat key to its 0-based position in `roles.judges`.
3070        let mut positions: Vec<usize> = absent
3071            .iter()
3072            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
3073            .collect();
3074        if positions.is_empty() {
3075            return Ok(false);
3076        }
3077        positions.sort_unstable();
3078        positions.dedup();
3079
3080        // Re-rank the lost seats, one blind prompt each.
3081        let mut judge_jobs = Vec::new();
3082        for &j in &positions {
3083            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3084            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3085            let seat_key = format!("judge-{}", j + 1);
3086            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3087            let seat = self.seat(&seat_key, &spec.id);
3088            judge_jobs.push(SeatJob {
3089                spec,
3090                seat,
3091                prompt: prompt::judge(
3092                    &self.state.instruction,
3093                    &views,
3094                    self.roles.judges.len(),
3095                    &base_short,
3096                    &language,
3097                ),
3098                cwd: root.join(seat_key),
3099                timeout,
3100                allow_write: false,
3101                sessions,
3102                artifacts: artifacts.clone(),
3103                stem: format!("judge-{}-recover", j + 1),
3104                handover: None,
3105            });
3106        }
3107
3108        let labels_for_check = labels.clone();
3109        let mut judge_losses = Vec::new();
3110        let retries = self.state.config.graph.retries;
3111        let cache = self.state.config.cache_dir();
3112        let ctx = WaveCtx {
3113            carry_seats: false,
3114            run: &run_id,
3115            node: "judge",
3116            prompts: &prompts,
3117            cache: cache.as_deref(),
3118            round: None,
3119        };
3120        let results = ask_json_wave::<Ranking>(
3121            judge_jobs,
3122            Arc::clone(&self.sem),
3123            retries,
3124            &self.roles.judge_roster,
3125            &ctx,
3126            &mut judge_losses,
3127            &mut self.state,
3128            &move |r: &Ranking| r.validate(&labels_for_check),
3129        )
3130        .await;
3131
3132        // Refresh the judgement of every seat that ranked again.
3133        let mut recovered: BTreeSet<usize> = BTreeSet::new();
3134        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
3135            let agent_id = seat.agent.clone();
3136            self.state.seats.insert(seat.key.clone(), seat);
3137            let record = &mut self.state.judgements[j];
3138            match res {
3139                Ok((ranking, out)) => {
3140                    record.agent = agent_id;
3141                    record.ranking = ranking.normalized();
3142                    record.reasons = ranking.reasons;
3143                    record.confidence = ranking.confidence;
3144                    record.failed = None;
3145                    record.duration_ms = out.duration_ms;
3146                    recovered.insert(j);
3147                    self.state.event(
3148                        "recover",
3149                        format!("judge {} ranked again after the limit", j + 1),
3150                    );
3151                }
3152                Err(e) => {
3153                    self.state
3154                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
3155                }
3156            }
3157        }
3158
3159        // Re-ask the votes of the seats that recovered a ranking.
3160        let mut vote_jobs = Vec::new();
3161        let mut vote_pos: Vec<usize> = Vec::new();
3162        for &j in &recovered {
3163            let seat_key = format!("judge-{}", j + 1);
3164            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
3165            let seat = self.seat(&seat_key, &spec.id);
3166            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
3167            let text = if has_context(&spec, &seat, sessions) {
3168                prompt::final_vote(&labels, &language)
3169            } else {
3170                full.clone()
3171            };
3172            vote_jobs.push(SeatJob {
3173                spec,
3174                seat,
3175                prompt: text,
3176                cwd: root.join(seat_key),
3177                timeout,
3178                allow_write: false,
3179                sessions,
3180                artifacts: artifacts.clone(),
3181                stem: format!("vote-judge-{}-recover", j + 1),
3182                handover: Some(full),
3183            });
3184            vote_pos.push(j);
3185        }
3186        let allowed = labels.clone();
3187        let mut vote_losses = Vec::new();
3188        let vote_retries = self.state.config.graph.retries;
3189        let vote_cache = self.state.config.cache_dir();
3190        let ctx = WaveCtx {
3191            carry_seats: false,
3192            run: &run_id,
3193            node: "vote",
3194            prompts: &prompts,
3195            cache: vote_cache.as_deref(),
3196            round: None,
3197        };
3198        let votes = ask_json_wave::<FinalVote>(
3199            vote_jobs,
3200            Arc::clone(&self.sem),
3201            vote_retries,
3202            &self.roles.judge_roster,
3203            &ctx,
3204            &mut vote_losses,
3205            &mut self.state,
3206            &move |v: &FinalVote| match v.label() {
3207                Some(c) if allowed.contains(&c) => Ok(()),
3208                other => bail!("vote {other:?} is not one of {allowed:?}"),
3209            },
3210        )
3211        .await;
3212        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
3213            let agent_id = seat.agent.clone();
3214            self.state.seats.insert(seat.key.clone(), seat);
3215            match res {
3216                Ok((v, _)) => {
3217                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
3218                        rec.vote = v.label();
3219                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
3220                    } else {
3221                        self.state.votes.push(VoteRecord {
3222                            judge: j + 1,
3223                            agent: agent_id,
3224                            vote: v.label(),
3225                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
3226                            changed: false,
3227                        });
3228                    }
3229                    self.state.event(
3230                        "recover",
3231                        format!("judge {} voted again after the limit", j + 1),
3232                    );
3233                }
3234                Err(e) => {
3235                    self.state
3236                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
3237                }
3238            }
3239        }
3240
3241        // A seat that ranked again is present even if its re-vote failed —
3242        // `tally` falls back to the initial ranking's first choice — so clear
3243        // its quota loss. Seats that still fail keep theirs and stay absent.
3244        let recovered_keys: BTreeSet<String> = recovered
3245            .iter()
3246            .map(|&j| format!("judge-{}", j + 1))
3247            .collect();
3248        self.state
3249            .quota
3250            .retain(|q| !recovered_keys.contains(&q.seat));
3251        // A seat that hit the limit again is a fresh loss, not the old one:
3252        // replace the stale entry so the history stays one-per-seat and the
3253        // daemon can tell this attempt's loss from a previous session's.
3254        for loss in judge_losses.into_iter().chain(vote_losses) {
3255            if recovered_keys.contains(&loss.seat) {
3256                continue;
3257            }
3258            self.state.quota.retain(|q| q.seat != loss.seat);
3259            self.state.quota.push(loss);
3260        }
3261
3262        // Recompute the verdict from the refreshed panel.
3263        self.state.tally = None;
3264        self.tally()?;
3265        Ok(self
3266            .state
3267            .tally
3268            .as_ref()
3269            .map(|t| t.met_quorum)
3270            .unwrap_or(false))
3271    }
3272
3273    // ----------------------------------------------------------------- fold
3274
3275    async fn fold_losers(&mut self) -> Result<()> {
3276        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
3277            return Ok(());
3278        };
3279        let repo = self.state.repo.clone();
3280        let mut folded = Vec::new();
3281        for i in 0..self.state.candidates.len() {
3282            let c = &self.state.candidates[i];
3283            if c.label == winner || c.folded {
3284                continue;
3285            }
3286            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
3287            git::worktree_remove(&repo, &wt).await.ok();
3288            git::branch_delete(&repo, &branch).await.ok();
3289            self.state.candidates[i].folded = true;
3290            folded.push(label.to_string());
3291        }
3292        // The judges are finished; their checkouts are pure cost from here.
3293        let root = self.state.worktree_root();
3294        for j in 1..=self.roles.judges.len() {
3295            let wt = root.join(format!("judge-{j}"));
3296            if wt.exists() {
3297                git::worktree_remove(&repo, &wt).await.ok();
3298            }
3299        }
3300        // The design-deliberation stage is finished by the time a tally
3301        // exists — same reasoning as the judges above.
3302        if self.state.config.graph.advise {
3303            for k in 1..=self.state.config.graph.advisors {
3304                let wt = root.join(format!("advisor-{k}"));
3305                if wt.exists() {
3306                    git::worktree_remove(&repo, &wt).await.ok();
3307                }
3308            }
3309        }
3310        if !folded.is_empty() {
3311            self.state
3312                .event("fold", format!("folded candidates {}", folded.join(", ")));
3313            self.state.save()?;
3314        }
3315        Ok(())
3316    }
3317
3318    // ------------------------------------------------------------ base sync
3319
3320    /// Land the winner's tree on the current tip of `<remote>/<base>` before
3321    /// anything verifies it.
3322    ///
3323    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
3324    /// read whatever is checked out in the winner's worktree. Left alone that
3325    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
3326    /// it when the run *branched* - and a run takes long enough that the base
3327    /// has usually moved by the time it gets here. A gate that ran there
3328    /// answers "green on the commit this run started from", not "green on
3329    /// what is about to land", and the difference showed up three times in
3330    /// one day as a green run whose merge would have reverted a file another
3331    /// pull request had already landed.
3332    ///
3333    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
3334    /// `land::Step::Rebase` calls, rather than a second implementation of the
3335    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
3336    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
3337    /// doc warns about ("two answers to a question nobody notices until a
3338    /// diff is wrong").
3339    ///
3340    /// A conflict is not the end of the road: the standing rebase is handed
3341    /// to the fixer seat, at most `graph.review_rounds` times, counted in
3342    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
3343    /// land). Once it finishes, review and the gate run as usual on the
3344    /// rebased tree, which is where a breakage the new base caused is caught
3345    /// by the ordinary gate-fix round. magi resolves nothing itself.
3346    ///
3347    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
3348    /// `state.base_sync.attempts`, is how many times the base is *rebased
3349    /// onto* (a base that keeps moving); `rebase_fixes` is how many times a
3350    /// *conflict* was given to a fixer. When the fixer cannot finish the
3351    /// rebase the branch is restored, `state.base_sync.conflict` is set with
3352    /// what was tried (rounds spent, paths still conflicted) and the branch
3353    /// and worktree stay exactly as they were - untouched, for a person to
3354    /// look at - which is also what makes re-entering this function
3355    /// afterwards a no-op instead of a second attempt at the same wall. A
3356    /// push failure ends the same way.
3357    async fn sync_to_base(&mut self) -> Result<()> {
3358        if self.state.status == RunStatus::AlreadyInBase {
3359            return Ok(());
3360        }
3361        let conflicted = self
3362            .state
3363            .base_sync
3364            .as_ref()
3365            .is_some_and(|s| s.conflict.is_some());
3366        let Some(winner) = self.state.winner().cloned() else {
3367            return Ok(());
3368        };
3369
3370        let repo = self.state.repo.clone();
3371        let remote = self.state.config.merge.remote.clone();
3372        let base_branch = self.state.base_branch.clone();
3373        let tracking = format!("{remote}/{base_branch}");
3374
3375        git::fetch(&repo, &remote, &base_branch).await.ok();
3376        // No network, or the remote never had this branch: `resolve_base`
3377        // already treats that as non-fatal at branch time, and a run that got
3378        // this far must not be blocked by it here either.
3379        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
3380            return Ok(());
3381        };
3382
3383        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3384        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
3385        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
3386
3387        // Before any rebase, and before a recorded conflict is honoured: a
3388        // branch whose change reached the base under other commit ids has
3389        // nothing to rebase and nothing to conflict with, and a run that
3390        // already stopped on that phantom conflict recovers here on resume.
3391        // `behind == 0` with head == tip is a branch the base has since taken
3392        // in whole, whether or not a conflict was ever recorded: the ancestry
3393        // proof must still run (`classify` ignores a head still on the start
3394        // commit).
3395        if (behind > 0 || conflicted || head == tip)
3396            && self
3397                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
3398                .await?
3399        {
3400            return Ok(());
3401        }
3402        if conflicted {
3403            return Ok(());
3404        }
3405
3406        if behind == 0 {
3407            // A fixer-finished rebase moves the branch ref before the
3408            // winner's worktree is told (`sync_to_head` below). A run that
3409            // died in between resumes here with `behind == 0` and a tree still
3410            // holding the pre-rebase files, which review and the gate would
3411            // then read. That state is exactly: HEAD moved off the tip the
3412            // rebase started from, yet the tree is still identical to that
3413            // tip. A tree with edits of its own differs from it, so nothing
3414            // is thrown away.
3415            if let Some(from) = self
3416                .state
3417                .rebase_fixes
3418                .iter()
3419                .rev()
3420                .find_map(|r| r.from.clone())
3421                && from != head
3422                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
3423                    .await
3424                    .is_ok_and(|o| o.ok())
3425            {
3426                git::sync_to_head(&winner.worktree).await?;
3427            }
3428            // An earlier attempt may have rebased the branch locally and died
3429            // before pushing it (only the fresh-rebase arm below pushes).
3430            // Publish it now, so the plain push at PR time is not refused as
3431            // a non-fast-forward. A run already holding a recorded conflict
3432            // never reaches here; that case is out of scope.
3433            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
3434            if let Some(why) = &conflict {
3435                self.state.status = RunStatus::Blocked;
3436                self.state.event("land", why.clone());
3437            }
3438            self.state.base_sync = Some(BaseSync {
3439                tip,
3440                behind: 0,
3441                attempts,
3442                conflict,
3443                already_in: None,
3444            });
3445            self.state.save()?;
3446            return Ok(());
3447        }
3448
3449        if attempts >= BASE_SYNC_ROUNDS {
3450            let why = format!(
3451                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
3452                 rebase(s); rebasing again would only race it",
3453                winner.branch
3454            );
3455            self.state.status = RunStatus::Blocked;
3456            self.state.base_sync = Some(BaseSync {
3457                tip,
3458                behind,
3459                attempts,
3460                conflict: Some(why.clone()),
3461                already_in: None,
3462            });
3463            self.state.event("land", why);
3464            self.state.save()?;
3465            return Ok(());
3466        }
3467
3468        self.state.event(
3469            "land",
3470            format!(
3471                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
3472                winner.branch
3473            ),
3474        );
3475        self.state.save()?;
3476
3477        // The remote's copy of the branch, read now and only if the fetch
3478        // really succeeded (a stale tracking ref must never pin a lease). It is
3479        // pushed over after a rebase only when it is a commit this branch
3480        // already contains, by ancestry or by patch (an earlier rebase of ours
3481        // that never reached the remote): anything else is somebody else's work.
3482        let branch_tracking = format!("{remote}/{}", winner.branch);
3483        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
3484        let remote_tip = if matches!(&fetched_branch, Ok(o) if o.ok()) {
3485            git::rev_parse(&repo, &branch_tracking).await.ok()
3486        } else {
3487            None
3488        };
3489        // A remote tip this branch does not contain is somebody else's work:
3490        // rebasing would leave a local tip that can never be pushed. Stop
3491        // before touching anything and say so.
3492        if let Some(theirs) = &remote_tip
3493            && !git::is_ancestor(&repo, theirs, &head).await
3494            && !crate::reconcile::origin_missing(&repo, &head, theirs)
3495                .await
3496                .is_ok_and(|missing| missing.is_empty())
3497        {
3498            let why = format!(
3499                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
3500                 them",
3501                short(theirs),
3502                winner.branch
3503            );
3504            self.state.status = RunStatus::Blocked;
3505            self.state.base_sync = Some(BaseSync {
3506                tip,
3507                behind,
3508                attempts,
3509                conflict: Some(why.clone()),
3510                already_in: None,
3511            });
3512            self.state.event("land", why);
3513            self.state.save()?;
3514            return Ok(());
3515        }
3516
3517        let scratch = self.state.dir().join("base-sync");
3518        let rebased = match crate::rebase::rebase_with_fixer(
3519            &mut self.state,
3520            &scratch,
3521            &winner.branch,
3522            &tracking,
3523        )
3524        .await
3525        {
3526            Ok(crate::rebase::Rebased::Applied) => Ok(None),
3527            Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
3528            Err(e) => Err(e),
3529        };
3530        let attempts = attempts + 1;
3531        match rebased {
3532            Ok(None) => {
3533                // The branch ref moved, but a worktree that already had it
3534                // checked out (the winner's) was not told; sync its index and
3535                // files before anything reads them.
3536                git::sync_to_head(&winner.worktree).await?;
3537                let mut conflict = None;
3538                if let Some(pinned) = &remote_tip {
3539                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
3540                    match pushed {
3541                        Ok(o) if o.ok() => self.state.event(
3542                            "land",
3543                            format!("pushed rebased {} to {remote}", winner.branch),
3544                        ),
3545                        Ok(o) => {
3546                            conflict = Some(format!(
3547                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
3548                                winner.branch,
3549                                short(pinned),
3550                                o.stderr.chars().take(600).collect::<String>()
3551                            ));
3552                        }
3553                        Err(e) => {
3554                            conflict = Some(format!(
3555                                "rebased {} locally but could not push it: {e:#}",
3556                                winner.branch
3557                            ));
3558                        }
3559                    }
3560                }
3561                if let Some(why) = &conflict {
3562                    self.state.status = RunStatus::Blocked;
3563                    self.state.event("land", why.clone());
3564                }
3565                self.state.base_sync = Some(BaseSync {
3566                    tip: tip.clone(),
3567                    behind: 0,
3568                    attempts,
3569                    conflict,
3570                    already_in: None,
3571                });
3572                self.state
3573                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
3574            }
3575            Ok(Some(conflict)) => {
3576                let why = format!(
3577                    "{} conflicts with {tracking} and did not rebase: {}",
3578                    winner.branch,
3579                    conflict.chars().take(600).collect::<String>()
3580                );
3581                self.state.status = RunStatus::Blocked;
3582                self.state.base_sync = Some(BaseSync {
3583                    tip,
3584                    behind,
3585                    attempts,
3586                    conflict: Some(why.clone()),
3587                    already_in: None,
3588                });
3589                self.state.event("land", why);
3590            }
3591            Err(e) => {
3592                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
3593                self.state.status = RunStatus::Blocked;
3594                self.state.base_sync = Some(BaseSync {
3595                    tip,
3596                    behind,
3597                    attempts,
3598                    conflict: Some(why.clone()),
3599                    already_in: None,
3600                });
3601                self.state.event("land", why);
3602            }
3603        }
3604        self.state.save()?;
3605        Ok(())
3606    }
3607
3608    /// Push a branch an earlier attempt rebased locally but never published,
3609    /// pinned to the remote tip read right after a successful fetch. Returns
3610    /// the reason when the run must stop; `None` when there was nothing to do
3611    /// (no remote copy, the same tip, or a remote copy this branch already
3612    /// contains, which the PR-time push fast-forwards) or the push succeeded.
3613    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
3614        let repo = self.state.repo.clone();
3615        let remote = self.state.config.merge.remote.clone();
3616        let fetched = git::fetch(&repo, &remote, branch).await;
3617        if !matches!(&fetched, Ok(o) if o.ok()) {
3618            return None;
3619        }
3620        let branch_tracking = format!("{remote}/{branch}");
3621        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
3622        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
3623            return None;
3624        }
3625        if !crate::reconcile::origin_missing(&repo, head, &theirs)
3626            .await
3627            .is_ok_and(|missing| missing.is_empty())
3628        {
3629            return Some(format!(
3630                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
3631                 them",
3632                short(&theirs)
3633            ));
3634        }
3635        match git::push_pinned(&repo, &remote, branch, &theirs).await {
3636            Ok(o) if o.ok() => {
3637                self.state
3638                    .event("land", format!("pushed rebased {branch} to {remote}"));
3639                None
3640            }
3641            Ok(o) => Some(format!(
3642                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
3643                 someone may have pushed): {}",
3644                short(&theirs),
3645                o.stderr.chars().take(600).collect::<String>()
3646            )),
3647            Err(e) => Some(format!(
3648                "{branch} is rebased locally but could not be pushed: {e:#}"
3649            )),
3650        }
3651    }
3652
3653    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
3654    /// is already on `tip` under other commit ids ([`crate::already`]); returns
3655    /// whether it did.
3656    ///
3657    /// Checked only when the base is ahead of the branch. A failing check is
3658    /// "not proven" - the ordinary rebase path then decides - never a reason to
3659    /// stop the run.
3660    ///
3661    /// The remote copy of the branch is held to the same standard as the local
3662    /// one: if it carries a tip this worktree does not, that tip must itself be
3663    /// proven in the base, or nothing is settled (a pull request would
3664    /// otherwise be closed over commits nobody checked). The pull request is
3665    /// closed *before* the terminal status is saved; if that fails for a
3666    /// reason other than a refusal (no network, a `gh` error) the run is left
3667    /// `Blocked` with the reason as its conflict, which a resume retries -
3668    /// the same recovery a phantom conflict gets.
3669    async fn settle_already_in(
3670        &mut self,
3671        branch: &str,
3672        tip: &str,
3673        head: &str,
3674        attempts: usize,
3675        behind: usize,
3676    ) -> Result<bool> {
3677        let repo = self.state.repo.clone();
3678        let remote = self.state.config.merge.remote.clone();
3679        let start = self.state.base_commit.clone();
3680        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
3681            Ok(Some(e)) => e,
3682            Ok(None) => return Ok(false),
3683            Err(e) => {
3684                tracing::warn!("already-in-base check for {branch}: {e:#}");
3685                return Ok(false);
3686            }
3687        };
3688        let mut verified = vec![head.to_owned()];
3689        let fetched = git::fetch(&repo, &remote, branch).await;
3690        if matches!(&fetched, Ok(o) if o.ok())
3691            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{branch}")).await
3692            && theirs != head
3693        {
3694            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
3695                Ok(Some(_)) => verified.push(theirs),
3696                _ => return Ok(false),
3697            }
3698        }
3699        let base_branch = self.state.base_branch.clone();
3700        let message = format!(
3701            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
3702             land",
3703            evidence.names(),
3704            evidence.proof.as_str()
3705        );
3706        let closed =
3707            crate::land::close_superseded_pr(&mut self.state, branch, &evidence, &verified).await;
3708        match closed {
3709            Ok(Ok(url)) => self
3710                .state
3711                .event("land", format!("closed {url}: superseded on {base_branch}")),
3712            Ok(Err(why)) => self
3713                .state
3714                .event("land", format!("did not close a pull request: {why}")),
3715            Err(e) => {
3716                let why = format!(
3717                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
3718                     not be closed ({e:#}); resume to retry"
3719                );
3720                self.state.status = RunStatus::Blocked;
3721                self.state.base_sync = Some(BaseSync {
3722                    tip: tip.to_owned(),
3723                    behind,
3724                    attempts,
3725                    conflict: Some(why.clone()),
3726                    already_in: None,
3727                });
3728                self.state.event("land", why);
3729                self.state.save()?;
3730                return Ok(true);
3731            }
3732        }
3733        self.state.status = RunStatus::AlreadyInBase;
3734        self.state.base_sync = Some(BaseSync {
3735            tip: tip.to_owned(),
3736            behind,
3737            attempts,
3738            conflict: None,
3739            already_in: Some(evidence),
3740        });
3741        self.state.event("land", message);
3742        self.state.save()?;
3743        self.settle_questions();
3744        Ok(true)
3745    }
3746
3747    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
3748    /// last landed the winner on, once it has run, else the commit the run
3749    /// branched from.
3750    ///
3751    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
3752    /// and `vote` all happen before there is a winner to rebase, so they
3753    /// compare every candidate against the branch point on purpose, and a
3754    /// base that moves after they are already done cannot change an answer
3755    /// they already gave.
3756    fn landing_base(&self) -> String {
3757        self.state
3758            .base_sync
3759            .as_ref()
3760            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
3761    }
3762
3763    // ------------------------------------------------------- operator fix
3764
3765    /// Route specific, already-recorded review findings to a fixer for a
3766    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
3767    /// entry point.
3768    ///
3769    /// Distinct from `review_loop`'s own fix step in three ways: it never
3770    /// runs a reviewer wave, it never spends review-round budget, and what
3771    /// happened is recorded as an [`OperatorFixRequest`] appended to
3772    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
3773    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
3774    /// and vote must never be rewritten to look like a manufactured blocking
3775    /// verdict.
3776    ///
3777    /// Only meaningful once review has actually concluded: `Ready` (handed
3778    /// off with findings still open, or simply concluded clean while minor
3779    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
3780    /// gate failed). Everything else is refused: a run still in progress
3781    /// should simply be resumed, and a `Merged` run's branch has already
3782    /// landed — reopening *this* run's own record cannot change that, so the
3783    /// answer there is a fresh `magi review <branch>`.
3784    ///
3785    /// A real commit here re-verifies through a fresh, ordinary review-only
3786    /// run on the same branch ([`Self::review`]) rather than reopening this
3787    /// run's own `review_loop`: once any round in this run's history went
3788    /// clean, `review_conclusion` treats that as permanent by design (the
3789    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
3790    /// way to force one more genuine reviewer wave out of *this* run without
3791    /// either rewriting history or weakening that guarantee for every other
3792    /// caller. A review-only run costs nothing extra — no implementation, no
3793    /// judging, no vote — and exercises the exact same review → verify →
3794    /// gate → (human) merge path, unmodified.
3795    pub async fn fix_selected(
3796        &mut self,
3797        ids: &[String],
3798        reason: &str,
3799        allow_stale: bool,
3800    ) -> Result<()> {
3801        let reason = reason.trim();
3802        if reason.is_empty() {
3803            bail!("a fix request needs a reason — that is the operator's own record of why");
3804        }
3805        if ids.is_empty() {
3806            bail!("no finding id given");
3807        }
3808        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
3809            bail!(
3810                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
3811                 has already concluded — can be given a targeted fix. A run still \
3812                 in progress should simply be resumed; a `merged` run's branch has \
3813                 already landed, so its answer is a fresh `magi review <branch>`, \
3814                 not reopening this run's own record",
3815                self.state.id,
3816                self.state.status.as_str()
3817            );
3818        }
3819        let Some(winner) = self.state.winner().cloned() else {
3820            bail!("run {} has no winning candidate to fix", self.state.id);
3821        };
3822        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
3823            bail!(
3824                "branch `{}` no longer exists; this run cannot be extended",
3825                winner.branch
3826            );
3827        }
3828        let home = crate::run::home();
3829        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
3830            bail!(
3831                "run {} is currently being worked on by another magi process",
3832                self.state.id
3833            );
3834        }
3835        // Held for the rest of this call, including the follow-up review
3836        // below: two `magi fix` invocations against the same run must not
3837        // both reach the worktree manipulation further down, which would
3838        // otherwise race to remove and recreate the same directory — see
3839        // [`FixClaim`]'s own doc.
3840        let _claim = FixClaim::acquire(&self.state.dir())?;
3841
3842        // Resolve every id before spending anything — an unknown id refuses
3843        // the whole request rather than silently dropping it — and dedup
3844        // while keeping the operator's own order.
3845        let mut seen = BTreeSet::new();
3846        let mut findings = Vec::new();
3847        let mut missing = Vec::new();
3848        for id in ids {
3849            if !seen.insert(id.clone()) {
3850                continue;
3851            }
3852            match self.state.finding(id) {
3853                Some((round, rec, f)) => findings.push(OperatorFixFinding {
3854                    id: f.id.clone(),
3855                    severity: f.severity,
3856                    reviewer_vote: rec.vote,
3857                    round: round.round,
3858                    round_head: round.head.clone(),
3859                    reviewer: rec.reviewer,
3860                    agent: rec.agent.clone(),
3861                    file: f.file.clone(),
3862                    line: f.line,
3863                    title: f.title.clone(),
3864                    detail: f.detail.clone(),
3865                    outcome: OperatorFixOutcome::Pending,
3866                }),
3867                None => missing.push(id.clone()),
3868            }
3869        }
3870        if !missing.is_empty() {
3871            bail!(
3872                "unknown finding id(s): {}; nothing was changed",
3873                missing.join(", ")
3874            );
3875        }
3876
3877        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
3878        let stale_details: Vec<(String, String)> = findings
3879            .iter()
3880            .filter(|f| f.round_head != head_at_request)
3881            .map(|f| (f.id.clone(), f.round_head.clone()))
3882            .collect();
3883        let stale = !stale_details.is_empty();
3884        if stale && !allow_stale {
3885            bail!(
3886                "the branch has moved since some finding(s) were raised — {} — now \
3887                 at {}; pass --allow-stale to fix anyway, or re-run review first",
3888                stale_details
3889                    .iter()
3890                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
3891                    .collect::<Vec<_>>()
3892                    .join(", "),
3893                short(&head_at_request)
3894            );
3895        }
3896
3897        let request = OperatorFixRequest {
3898            requested_at: Timestamp::now(),
3899            reason: reason.to_owned(),
3900            findings,
3901            head_at_request: head_at_request.clone(),
3902            allow_stale,
3903            stale,
3904            fix: None,
3905            result_head: None,
3906            follow_up_review_run: None,
3907        };
3908        self.state.event(
3909            "fix",
3910            format!(
3911                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
3912                request.findings.len(),
3913                request
3914                    .findings
3915                    .iter()
3916                    .map(|f| f.id.as_str())
3917                    .collect::<Vec<_>>()
3918                    .join(", "),
3919            ),
3920        );
3921        // Recorded now, before any worktree work or the fixer call itself —
3922        // and re-saved at each checkpoint below: a crash at any point after
3923        // this (mid fixer call, mid follow-up review) must not lose the fact
3924        // that this was requested, for which findings, and why. Everything
3925        // past this point reads and writes through `request_index` rather
3926        // than a local variable, since `request` itself is moved here.
3927        self.state.operator_fixes.push(request);
3928        self.state.save()?;
3929        let request_index = self.state.operator_fixes.len() - 1;
3930
3931        // A fresh, dedicated worktree for this one call, never the winner's
3932        // own worktree in place: that one may already be gone (folded away),
3933        // and reusing it in place would leave the branch checked out there
3934        // when the follow-up review below tries to check it out again. Freed
3935        // immediately after, either way — but only once confirmed clean:
3936        // `worktree_remove` is a `git worktree remove --force`, which would
3937        // otherwise discard uncommitted work left there by the operator or
3938        // another process before this had a chance to even look at it.
3939        if winner.worktree.exists() {
3940            // Lockfiles a rescue commit withheld stay untracked on purpose and
3941            // are already recorded; they are not the operator's work to protect.
3942            let dirty = git::git(
3943                &winner.worktree,
3944                &["status", "--porcelain", "--untracked-files=all"],
3945            )
3946            .await?;
3947            let only_withheld = dirty.lines().all(|l| {
3948                l.strip_prefix("?? ")
3949                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
3950            });
3951            if !only_withheld {
3952                bail!(
3953                    "`{}` has uncommitted changes; refusing to touch it — commit or \
3954                     discard them first",
3955                    winner.worktree.display()
3956                );
3957            }
3958            git::worktree_remove(&self.state.repo, &winner.worktree)
3959                .await
3960                .ok();
3961        }
3962        let fix_worktree = self.state.worktree_root().join("operator-fix");
3963        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
3964        git::git(
3965            &self.state.repo,
3966            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
3967        )
3968        .await
3969        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
3970        if !git::is_clean(&fix_worktree).await? {
3971            git::worktree_remove(&self.state.repo, &fix_worktree)
3972                .await
3973                .ok();
3974            bail!(
3975                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
3976                winner.branch
3977            );
3978        }
3979
3980        let run_id = self.state.id.clone();
3981        let prompts = self.state.config.prompts.clone();
3982        let language = self.state.config.graph.language.clone();
3983        let sessions = self.state.config.graph.sessions;
3984        let artifacts = agent::artifacts_dir(&self.state.dir());
3985        let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3986            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3987            _ => (
3988                self.state
3989                    .config
3990                    .agent(&winner.agent)
3991                    .cloned()
3992                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3993                format!("impl-{}", winner.label),
3994            ),
3995        };
3996        let seat = self.seat(&fix_seat_key, &fix_spec.id);
3997        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
3998            .findings
3999            .iter()
4000            .map(|f| Finding {
4001                id: f.id.clone(),
4002                severity: f.severity,
4003                file: f.file.clone(),
4004                line: f.line,
4005                title: f.title.clone(),
4006                detail: f.detail.clone(),
4007            })
4008            .collect();
4009        let job = SeatJob {
4010            prompt: prompt::operator_fix(
4011                &self.state.instruction,
4012                &finding_list,
4013                reason,
4014                &stale_details,
4015                &head_at_request,
4016                &language,
4017            ),
4018            spec: fix_spec.clone(),
4019            seat,
4020            cwd: fix_worktree.clone(),
4021            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4022            allow_write: true,
4023            sessions,
4024            artifacts: artifacts.clone(),
4025            stem: "operator-fix".to_owned(),
4026            handover: None,
4027        };
4028        let cache = self.state.config.cache_dir();
4029        let ctx = WaveCtx {
4030            carry_seats: false,
4031            run: &run_id,
4032            node: "fix",
4033            prompts: &prompts,
4034            cache: cache.as_deref(),
4035            round: None,
4036        };
4037        let (seat, out) =
4038            run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4039        let agent_id = seat.agent.clone();
4040
4041        let mut fix = FixRecord {
4042            agent: agent_id,
4043            addressed: Vec::new(),
4044            rejected: Vec::new(),
4045            notes: String::new(),
4046            committed: false,
4047            failed: None,
4048            duration_ms: 0,
4049            continuation: None,
4050        };
4051        let mut final_seat = seat.clone();
4052        match out {
4053            AgentOutcome::Ok(o) => {
4054                fix.duration_ms = o.duration_ms;
4055                let parsed = verdict::extract_json::<FixReport>(&o.text);
4056                let incomplete_reason = match &parsed {
4057                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4058                        "the reply parsed, but it reported a command whose own CLI \
4059                         never confirmed an exit status"
4060                            .to_owned(),
4061                    ),
4062                    Ok(_) => None,
4063                    Err(e) => Some(e.to_string()),
4064                };
4065                match incomplete_reason {
4066                    None => {
4067                        let report = parsed.expect("checked Ok above");
4068                        fix.addressed = report.addressed;
4069                        fix.rejected = report.rejected;
4070                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
4071                    }
4072                    Some(reason) => {
4073                        let (resumed_seat, resolved, failure, cont) = self
4074                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
4075                            .await;
4076                        fix.duration_ms += cont.cumulative_wait_ms;
4077                        fix.continuation = Some(cont);
4078                        final_seat = resumed_seat;
4079                        match resolved {
4080                            Some(report) => {
4081                                fix.addressed = report.addressed;
4082                                fix.rejected = report.rejected;
4083                                fix.notes =
4084                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
4085                            }
4086                            None => fix.failed = failure,
4087                        }
4088                    }
4089                }
4090            }
4091            AgentOutcome::Dropped(o) => {
4092                fix.duration_ms = o.duration_ms;
4093                let why = o
4094                    .dropped
4095                    .as_ref()
4096                    .map(|d| d.why.as_str())
4097                    .unwrap_or("the CLI ended the stream without delivering its answer");
4098                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
4099            }
4100            AgentOutcome::Quota(o) => {
4101                self.state.quota.push(QuotaLoss {
4102                    seat: final_seat.key.clone(),
4103                    node: "fix".to_owned(),
4104                    at: Timestamp::now(),
4105                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4106                });
4107                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
4108            }
4109            AgentOutcome::Failed(e) => fix.failed = Some(e),
4110        }
4111        if fix.continuation.is_none() {
4112            fix.continuation = Some(ContinuationRecord::not_needed());
4113        }
4114        self.state.seats.insert(final_seat.key.clone(), final_seat);
4115
4116        let rescue_message = format!(
4117            "magi: operator-selected fix ({}) (uncommitted work)",
4118            self.state.operator_fixes[request_index]
4119                .findings
4120                .iter()
4121                .map(|f| f.id.as_str())
4122                .collect::<Vec<_>>()
4123                .join(", ")
4124        );
4125        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
4126            self.state.note_withheld("fix", &r.withheld);
4127        }
4128        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
4129        fix.committed = after != head_at_request;
4130        git::worktree_remove(&self.state.repo, &fix_worktree)
4131            .await
4132            .ok();
4133
4134        self.state.event(
4135            "fix",
4136            match &fix.failed {
4137                Some(reason) => format!(
4138                    "operator fix: adoption report was lost ({reason}); {}",
4139                    if fix.committed {
4140                        "committed"
4141                    } else {
4142                        "NO new commit"
4143                    }
4144                ),
4145                None => format!(
4146                    "operator fix: {} addressed, {} rejected, {}",
4147                    fix.addressed.len(),
4148                    fix.rejected.len(),
4149                    if fix.committed {
4150                        "committed"
4151                    } else {
4152                        "NO new commit"
4153                    }
4154                ),
4155            },
4156        );
4157
4158        // Every selected finding gets an outcome — never left `Pending` once
4159        // the fixer's own turn is over. A report that never came back at all
4160        // marks every one of them `Unreported`, not silently "not addressed":
4161        // quota, a dropped stream, or an exhausted continuation are gaps in
4162        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
4163        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
4164        for f in &mut self.state.operator_fixes[request_index].findings {
4165            f.outcome = if fix.failed.is_some() {
4166                OperatorFixOutcome::Unreported
4167            } else if fix.addressed.contains(&f.id) {
4168                OperatorFixOutcome::Addressed
4169            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
4170                OperatorFixOutcome::Rejected { why: r.why.clone() }
4171            } else {
4172                OperatorFixOutcome::Unreported
4173            };
4174        }
4175
4176        let committed = fix.committed;
4177        if committed {
4178            self.state.operator_fixes[request_index].result_head = Some(after.clone());
4179        }
4180        self.state.operator_fixes[request_index].fix = Some(fix);
4181        // Saved again now that the fixer's own outcome is final, on top of
4182        // the save right after the request was first pushed above.
4183        self.state.save()?;
4184
4185        if committed {
4186            self.state.event(
4187                "fix",
4188                format!(
4189                    "operator fix committed {}; opening a follow-up review-only run",
4190                    short(&after)
4191                ),
4192            );
4193            // The operator asked for the fix, and the follow-up serves whatever
4194            // task the run it follows served.
4195            let origin =
4196                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
4197            match Self::review(
4198                &self.state.repo,
4199                &winner.branch,
4200                self.state.config.clone(),
4201                origin,
4202            )
4203            .await
4204            {
4205                Ok(mut follow_up) => {
4206                    follow_up.state.event(
4207                        "start",
4208                        format!(
4209                            "requested by an operator fix on run {} for finding(s) {}",
4210                            self.state.id,
4211                            self.state.operator_fixes[request_index]
4212                                .findings
4213                                .iter()
4214                                .map(|f| f.id.as_str())
4215                                .collect::<Vec<_>>()
4216                                .join(", "),
4217                        ),
4218                    );
4219                    follow_up.state.save()?;
4220                    let follow_up_id = follow_up.state.id.clone();
4221                    if let Err(e) = follow_up.execute().await {
4222                        self.state.event(
4223                            "fix",
4224                            format!(
4225                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
4226                            ),
4227                        );
4228                    }
4229                    self.state.operator_fixes[request_index].follow_up_review_run =
4230                        Some(follow_up_id);
4231                }
4232                Err(e) => {
4233                    self.state.event(
4234                        "fix",
4235                        format!("committed the fix but could not open a follow-up review: {e:#}"),
4236                    );
4237                }
4238            }
4239            self.state.save()?;
4240        }
4241
4242        Ok(())
4243    }
4244
4245    // --------------------------------------------------------------- review
4246
4247    /// The agent and seat key that fix the winner's tree: the configured
4248    /// fixer, else the winner's own implementer seat, whose conversation
4249    /// continues now that the competition is over. Shared by the review loop
4250    /// and the gate-fix round so both talk to the same seat.
4251    fn fixer_spec(&self, winner: &Candidate) -> (AgentSpec, String) {
4252        match &self.roles.fixer {
4253            Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
4254            _ => (
4255                self.state
4256                    .config
4257                    .agent(&winner.agent)
4258                    .cloned()
4259                    .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
4260                format!("impl-{}", winner.label),
4261            ),
4262        }
4263    }
4264
4265    async fn review_loop(&mut self) -> Result<()> {
4266        // A base that would not rebase is a person's decision, not a review
4267        // round: nothing here would change the answer, and reviewers and a
4268        // fixer would be spending real budget on a tree that cannot land
4269        // regardless of what they find.
4270        if self
4271            .state
4272            .base_sync
4273            .as_ref()
4274            .is_some_and(|s| s.conflict.is_some())
4275        {
4276            return Ok(());
4277        }
4278        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4279        // agent files with `magi task add` name the run that paid for it. The
4280        // prompt overlay is cloned alongside it because the waves borrow it
4281        // while `self` is mutably borrowed by the node's own bookkeeping.
4282        let run_id = self.state.id.clone();
4283        let prompts = self.state.config.prompts.clone();
4284        let Some(winner) = self.state.winner().cloned() else {
4285            return Ok(());
4286        };
4287        let max_rounds = self.state.config.graph.review_rounds;
4288        // A clean round, an exhausted round budget, or a stalled tree (see
4289        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
4290        // they are recorded — recomputed here, not read off `status`, so a
4291        // reentry into a run that already stopped restates the identical
4292        // verdict instead of silently handing back whatever an earlier node
4293        // in this same walk clobbered `status` to (a solo-candidate
4294        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
4295        // below runs an empty range once the budget is spent, and would
4296        // otherwise fall through without touching `status` at all.
4297        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
4298            // A reentry after a crash between the last round's save and
4299            // `stop_reviewing` reaches the hand-off here, not there.
4300            if status == RunStatus::Gating {
4301                self.record_contested_handoff();
4302            }
4303            self.state.status = status;
4304            self.state.save()?;
4305            return Ok(());
4306        }
4307        self.state.status = RunStatus::Reviewing;
4308        // A last recorded round whose own verification never resolved
4309        // (`ResourceBlocked` — the shared build cache, not the patch) is
4310        // never a concluded round, whatever the round budget says: starting
4311        // a fresh round on top of it would spend a whole new reviewer wave
4312        // re-reading an unchanged patch instead of just retrying the one
4313        // check that actually needs it, and once the budget is spent the
4314        // loop below has nothing left to do at all (its range is empty).
4315        // Retry that check directly instead, exactly the same retry
4316        // `stop_reviewing` already does for its own catch-up case.
4317        if self
4318            .state
4319            .reviews
4320            .last()
4321            .is_some_and(|r| r.e2e_status() == E2eStatus::ResourceBlocked)
4322        {
4323            let shell = self.state.config.shell();
4324            return self
4325                .stop_reviewing(
4326                    "the last round's own verification never resolved",
4327                    &shell,
4328                    &winner.worktree,
4329                )
4330                .await;
4331        }
4332
4333        let repo = self.state.repo.clone();
4334        let root = self.state.worktree_root();
4335        let language = self.state.config.graph.language.clone();
4336        let sessions = self.state.config.graph.sessions;
4337        let artifacts = agent::artifacts_dir(&self.state.dir());
4338        let base = self.landing_base();
4339        let base_short = short(&base);
4340        let reviewers = self.roles.reviewers.clone();
4341        let shell = self.state.config.shell();
4342
4343        for round in (self.state.reviews.len() + 1)..=max_rounds {
4344            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
4345            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
4346            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
4347            // The prior round's own record, already persisted — never a
4348            // hand-carried variable of just its failing output: that is
4349            // exactly what let a round's e2e result drift out of sync with
4350            // which commit it was actually about (see `SCHEMA`'s doc for
4351            // schema 8). Judged against `head`, the commit reviewers are
4352            // about to look at now, so the summary always reads as "an
4353            // earlier head" here — this round's own patch has not been
4354            // checked yet.
4355            let prev_verification = self
4356                .state
4357                .reviews
4358                .last()
4359                .and_then(|r| r.verification_summary(&head));
4360
4361            // Each reviewer gets its own detached checkout of exactly this
4362            // commit: nobody can perturb the winner's tree, and the fixer can
4363            // keep working without racing a reviewer.
4364            let mut jobs = Vec::new();
4365            for (r, spec) in reviewers.iter().cloned().enumerate() {
4366                let wt = root.join(format!("review-{}", r + 1));
4367                if wt.exists() {
4368                    git::reset_detached(&wt, &head).await?;
4369                } else {
4370                    git::worktree_add_detached(&repo, &wt, &head).await?;
4371                }
4372                let seat_key = format!("review-{}", r + 1);
4373                // The seat starts the round on whoever answered it last, not
4374                // on the agent the spec names, so a failure is not re-paid.
4375                let spec = pick_start_spec(
4376                    &self.roles.reviewer_roster,
4377                    spec,
4378                    self.state.seat_history.get(&seat_key),
4379                );
4380                let seat = self.seat(&seat_key, &spec.id);
4381                jobs.push(SeatJob {
4382                    prompt: prompt::review(&prompt::ReviewCtx {
4383                        instruction: &self.state.instruction,
4384                        branch: &winner.branch,
4385                        base_short: &base_short,
4386                        stat: &stat,
4387                        patch: &patch,
4388                        verification: prev_verification.as_ref(),
4389                        reviewers: reviewers.len(),
4390                        round,
4391                        rounds: max_rounds,
4392                        // A review-only run has no rankings, so nothing
4393                        // competed for this patch and the reviewer is told so.
4394                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
4395                        lens: Lens::for_seat(r),
4396                        language: &language,
4397                    }),
4398                    spec,
4399                    seat,
4400                    cwd: wt,
4401                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4402                    allow_write: false,
4403                    sessions,
4404                    artifacts: artifacts.clone(),
4405                    stem: format!("review-{round}-{}", r + 1),
4406                    handover: None,
4407                });
4408            }
4409
4410            self.state.event(
4411                "review",
4412                format!(
4413                    "round {round}: {} reviewers on {}",
4414                    jobs.len(),
4415                    short(&head)
4416                ),
4417            );
4418            let mut quota_losses = Vec::new();
4419            let review_retries = self.state.config.graph.retries;
4420            let review_cache = self.state.config.cache_dir();
4421            let ctx = WaveCtx {
4422                carry_seats: true,
4423                run: &run_id,
4424                node: "review",
4425                prompts: &prompts,
4426                cache: review_cache.as_deref(),
4427                round: Some(round),
4428            };
4429            let results = ask_json_wave::<Review>(
4430                jobs,
4431                Arc::clone(&self.sem),
4432                review_retries,
4433                &self.roles.reviewer_roster,
4434                &ctx,
4435                &mut quota_losses,
4436                &mut self.state,
4437                &|_: &Review| Ok(()),
4438            )
4439            .await;
4440            // Counted before the move below: how many of *this* round's
4441            // reviewer seats were lost to their own rate limit, as opposed to
4442            // a crash, a timeout, or unparsable output — see `round_is_clean`.
4443            let round_quota_missing = quota_losses.len();
4444            self.state.quota.extend(quota_losses);
4445
4446            let mut records = Vec::new();
4447            let mut all_findings = Vec::new();
4448            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
4449                let agent_id = seat.agent.clone();
4450                self.state.seats.insert(seat.key.clone(), seat);
4451                let mut record = ReviewRecord {
4452                    reviewer: r + 1,
4453                    agent: agent_id,
4454                    summary: String::new(),
4455                    findings: Vec::new(),
4456                    vote: None,
4457                    failed: None,
4458                    duration_ms: 0,
4459                    // Set for both outcomes: `failed: Some(_)` with
4460                    // `attempts > 0` is a seat every retry still lost, not a
4461                    // recovered one — only `failed: None` with `attempts > 0`
4462                    // reads as "answered after a nudge" (see this field's own
4463                    // doc).
4464                    attempts,
4465                };
4466                match res {
4467                    Ok((review, out)) => {
4468                        // Sanitized here, at the point every other piece of
4469                        // agent prose in this file is (candidate summaries,
4470                        // deliberation turns, vote reasons): a reviewer's own
4471                        // words are the one thing about it that could name
4472                        // it, and reconsideration below broadcasts this same
4473                        // summary and these same findings to every other
4474                        // seat on the panel.
4475                        record.summary =
4476                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
4477                        record.vote = Some(review.vote);
4478                        record.duration_ms = out.duration_ms;
4479                        for (n, mut f) in review.findings.into_iter().enumerate() {
4480                            // ids are magi's, never the agent's: the fixer's
4481                            // adoption report is keyed by them.
4482                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
4483                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
4484                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
4485                            // `file` is agent-supplied prose too, never
4486                            // checked against the real tree — the same
4487                            // exposure `title`/`detail` above have, just in
4488                            // a field easy to forget because it looks like a
4489                            // path rather than free text.
4490                            f.file = f
4491                                .file
4492                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
4493                            all_findings.push(f.clone());
4494                            record.findings.push(f);
4495                        }
4496                        self.state.event(
4497                            "review",
4498                            format!(
4499                                "round {round}: reviewer {} voted {} with {} finding(s)",
4500                                r + 1,
4501                                review.vote.label(),
4502                                record.findings.len()
4503                            ),
4504                        );
4505                    }
4506                    Err(e) => {
4507                        record.failed = Some(e.to_string());
4508                        self.state.event(
4509                            "review",
4510                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
4511                        );
4512                    }
4513                }
4514                records.push(record);
4515            }
4516
4517            // Tally the round's votes and, if they split, spend the one
4518            // round of reconsideration the split -> deliberate -> revote
4519            // shape `judge`/`vote` use for the panel, sized down to what a
4520            // read-only review round can afford: one round, and a revote
4521            // rather than an argument, because the panel already wrote its
4522            // reasoning down as findings the first time around.
4523            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
4524            let vote_split =
4525                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
4526            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
4527            if vote_split {
4528                self.state.event(
4529                    "review",
4530                    format!(
4531                        "round {round}: votes split ({}) — one round of reconsideration",
4532                        initial_votes
4533                            .iter()
4534                            .map(|v| v.label())
4535                            .collect::<Vec<_>>()
4536                            .join(", ")
4537                    ),
4538                );
4539                // Seats read every seat's findings and votes, still numbered
4540                // and never named — the same anonymity `review` itself keeps.
4541                let panel: Vec<ReviewSeatReport<'_>> = records
4542                    .iter()
4543                    .filter_map(|r| {
4544                        r.vote.map(|vote| ReviewSeatReport {
4545                            reviewer: r.reviewer,
4546                            vote,
4547                            summary: &r.summary,
4548                            findings: &r.findings,
4549                        })
4550                    })
4551                    .collect();
4552
4553                let mut jobs = Vec::new();
4554                let mut seats_at = Vec::new();
4555                for (r, spec) in reviewers.iter().cloned().enumerate() {
4556                    // A seat with no initial vote has nothing to reconsider
4557                    // from and stays absent, the same as it stayed absent
4558                    // from `panel` above.
4559                    if records[r].vote.is_none() {
4560                        continue;
4561                    }
4562                    let wt = root.join(format!("review-{}", r + 1));
4563                    let seat_key = format!("review-{}", r + 1);
4564                    let spec = self.occupant(&seat_key, spec);
4565                    let seat = self.seat(&seat_key, &spec.id);
4566                    // A seat with no live session has already forgotten the
4567                    // initial review's prompt — restate the patch it is
4568                    // voting on, the same as `deliberate`/`vote` do for a
4569                    // judge in the same position.
4570                    // The panel already carries this seat's own review and
4571                    // vote, so restating the patch makes the prompt whole for
4572                    // a seat handed to another agent.
4573                    let build = |with_patch: bool| {
4574                        prompt::review_reconsider(&ReviewReconsiderCtx {
4575                            instruction: &self.state.instruction,
4576                            reviewer: r + 1,
4577                            lens: Lens::for_seat(r),
4578                            panel: &panel,
4579                            patch: with_patch.then_some(ReviewPatch {
4580                                branch: &winner.branch,
4581                                base_short: &base_short,
4582                                stat: &stat,
4583                                patch: &patch,
4584                            }),
4585                            round,
4586                            rounds: max_rounds,
4587                            language: &language,
4588                        })
4589                    };
4590                    let full = build(true);
4591                    let prompt = if has_context(&spec, &seat, sessions) {
4592                        build(false)
4593                    } else {
4594                        full.clone()
4595                    };
4596                    jobs.push(SeatJob {
4597                        prompt,
4598                        spec,
4599                        seat,
4600                        cwd: wt,
4601                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
4602                        allow_write: false,
4603                        sessions,
4604                        artifacts: artifacts.clone(),
4605                        stem: format!("review-{round}-reconsider-{}", r + 1),
4606                        handover: Some(full),
4607                    });
4608                    seats_at.push(r);
4609                }
4610
4611                let mut recon_quota_losses = Vec::new();
4612                let recon_cache = self.state.config.cache_dir();
4613                let recon_ctx = WaveCtx {
4614                    carry_seats: true,
4615                    run: &run_id,
4616                    node: "review",
4617                    prompts: &prompts,
4618                    cache: recon_cache.as_deref(),
4619                    round: Some(round),
4620                };
4621                let recon_results = ask_json_wave::<ReviewRevote>(
4622                    jobs,
4623                    Arc::clone(&self.sem),
4624                    review_retries,
4625                    &self.roles.reviewer_roster,
4626                    &recon_ctx,
4627                    &mut recon_quota_losses,
4628                    &mut self.state,
4629                    &|_: &ReviewRevote| Ok(()),
4630                )
4631                .await;
4632                self.state.quota.extend(recon_quota_losses);
4633
4634                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
4635                    let agent_id = seat.agent.clone();
4636                    self.state.seats.insert(seat.key.clone(), seat);
4637                    let mut rec = ReviewRevoteRecord {
4638                        reviewer: r + 1,
4639                        agent: agent_id,
4640                        vote: None,
4641                        reason: String::new(),
4642                        failed: None,
4643                    };
4644                    match res {
4645                        Ok((rv, _)) => {
4646                            rec.vote = Some(rv.vote);
4647                            rec.reason =
4648                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
4649                            self.state.event(
4650                                "review",
4651                                format!(
4652                                    "round {round}: reviewer {} revoted {}",
4653                                    r + 1,
4654                                    rv.vote.label()
4655                                ),
4656                            );
4657                        }
4658                        Err(e) => {
4659                            rec.failed = Some(e.to_string());
4660                            self.state.event(
4661                                "review",
4662                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
4663                            );
4664                        }
4665                    }
4666                    reconsideration.push(rec);
4667                }
4668            } else if initial_votes.len() > 1 {
4669                self.state.event(
4670                    "review",
4671                    format!(
4672                        "round {round}: votes agreed ({}) — no reconsideration",
4673                        initial_votes[0].label()
4674                    ),
4675                );
4676            }
4677
4678            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
4679            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
4680            // A round that already has a blocking finding and a round left to
4681            // try is going back to the fixer no matter what `verify.e2e`
4682            // says, so running it first only spends the loop's slowest step
4683            // (minutes, for a Rust repo's full test suite) on a head about
4684            // to be rewritten. Deferred, never skipped: `verify.e2e` still
4685            // runs once a round has no blocking findings left (see
4686            // `round_is_clean`, which a deferred — empty — `e2e` can never
4687            // satisfy since `blocking` is nonzero whenever this branch is
4688            // taken), and `stop_reviewing` forces a real run before it will
4689            // ever read a deferred round as green.
4690            let defer_e2e =
4691                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
4692            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
4693                let reason =
4694                    format!("{blocking} blocking finding(s) already required a fix this round");
4695                self.state.event(
4696                    "verify",
4697                    format!(
4698                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
4699                         {}); it will run once a round has none left",
4700                        short(&head)
4701                    ),
4702                );
4703                (Vec::new(), false, true, Some(reason))
4704            } else {
4705                let e2e_commands = self.state.config.verify.e2e.clone();
4706                let cache_dir = self.state.config.cache_dir();
4707                let context = format!("round {round}");
4708                let (e2e, verify_retried) = with_cache_lease(
4709                    &mut self.state,
4710                    cache_dir.as_deref(),
4711                    "e2e",
4712                    "e2e",
4713                    &winner.worktree,
4714                    &head,
4715                    verify_timeout,
4716                    &context,
4717                    |state, budget| {
4718                        let shell = shell.clone();
4719                        let e2e_commands = e2e_commands.clone();
4720                        let worktree = winner.worktree.clone();
4721                        let context = context.clone();
4722                        async move {
4723                            run_e2e_with_retry(
4724                                state,
4725                                &shell,
4726                                &e2e_commands,
4727                                &worktree,
4728                                budget,
4729                                &context,
4730                            )
4731                            .await
4732                        }
4733                    },
4734                )
4735                .await;
4736                (e2e, verify_retried, false, None)
4737            };
4738
4739            let expected = records.len();
4740            let answered = records.iter().filter(|r| r.failed.is_none()).count();
4741            let incomplete = answered < expected;
4742            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
4743            let policy = self.state.config.graph.incomplete_review;
4744            let clean = round_is_clean(
4745                blocking,
4746                e2e_ok,
4747                answered,
4748                expected,
4749                round_quota_missing,
4750                policy,
4751            );
4752
4753            let mut round_record = ReviewRound {
4754                round,
4755                head: head.clone(),
4756                verified_head: None,
4757                verified_at: None,
4758                reviews: records,
4759                e2e,
4760                verify_retried,
4761                e2e_deferred,
4762                e2e_defer_reason,
4763                fix: None,
4764                blocking,
4765                answered,
4766                expected,
4767                clean,
4768                progressed: false,
4769                vote_split,
4770                reconsideration,
4771                verdict: None,
4772            };
4773            // The final vote per seat is its revote where reconsideration
4774            // ran and answered, its initial vote otherwise — the same
4775            // fallback `tally` uses for a judge whose private vote failed.
4776            round_record.verdict = ReviewVote::worst(
4777                round_record
4778                    .final_votes()
4779                    .into_iter()
4780                    .map(|(_, _, vote)| vote),
4781            );
4782            // Which commit and when magi actually attempted to check —
4783            // known the moment a command was dispatched against `head`,
4784            // whether or not it finished: a resource-blocked attempt still
4785            // targeted a specific commit at a specific time, and leaving
4786            // that unrecorded is exactly what made `verification_summary`
4787            // report a fresh attempt as "commit unknown ... recorded before
4788            // this was tracked", indistinguishable from a genuinely old,
4789            // untracked record. Only a deferred or unconfigured round never
4790            // ran at all and has nothing to record — see
4791            // `ReviewRound::verified_head`'s own doc.
4792            if !matches!(
4793                round_record.e2e_status(),
4794                E2eStatus::Deferred | E2eStatus::NotConfigured
4795            ) {
4796                round_record.verified_head = Some(head.clone());
4797                round_record.verified_at = Some(Timestamp::now());
4798            }
4799            let this_round_verification = round_record.verification_summary(&head);
4800
4801            if incomplete {
4802                let missing: Vec<String> = round_record
4803                    .reviews
4804                    .iter()
4805                    .filter(|r| r.failed.is_some())
4806                    .map(|r| format!("review-{}", r.reviewer))
4807                    .collect();
4808                self.state.event(
4809                    "review",
4810                    format!(
4811                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
4812                        missing.join(", ")
4813                    ),
4814                );
4815            }
4816
4817            if clean {
4818                self.state.event(
4819                    "review",
4820                    if incomplete && policy == IncompleteReviewPolicy::Warn {
4821                        format!(
4822                            "round {round}: clean (warn policy, incomplete panel) — no \
4823                             blocking findings from the seats that answered, verification green"
4824                        )
4825                    } else if incomplete {
4826                        format!(
4827                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
4828                             quorum) — no blocking findings from the seats that answered, \
4829                             verification green",
4830                            expected - answered
4831                        )
4832                    } else {
4833                        format!("round {round}: clean — no blocking findings, verification green")
4834                    },
4835                );
4836                self.state.reviews.push(round_record);
4837                self.state.status = RunStatus::Gating;
4838                self.state.save()?;
4839                return Ok(());
4840            }
4841
4842            // Nothing was raised and verification passed, but not every seat
4843            // answered and `round_is_clean` still refused to call it clean —
4844            // either a seat is missing for a reason other than its own quota
4845            // (a crash, a timeout, unparsable output — worth another try), or
4846            // every seat that could have answered lost its quota and nobody
4847            // is left to decide on: re-review rather than send the fixer
4848            // after a round with nothing to fix.
4849            if incomplete && blocking == 0 && e2e_ok {
4850                self.state.reviews.push(round_record);
4851                self.state.save()?;
4852                if round == max_rounds {
4853                    self.state.status = RunStatus::Blocked;
4854                    self.state.event(
4855                        "review",
4856                        format!(
4857                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
4858                             refusing to call it clean",
4859                            expected - answered
4860                        ),
4861                    );
4862                    return Ok(());
4863                }
4864                continue;
4865            }
4866
4867            // Nothing for the fixer to act on (`blocking == 0`) and the only
4868            // reason this round is not clean is that magi itself never got
4869            // a command to run — the shared build cache, not the patch (see
4870            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
4871            // the fixer would invite a change to appease contention that has
4872            // nothing to do with the diff, and would leave this attempt
4873            // sitting in the next round's prompt as if it were about an
4874            // earlier, superseded commit rather than what it actually is:
4875            // the same head, still waiting to be checked. Wait for it the
4876            // same way the final round's own contention is already handled,
4877            // whatever round this happens to be.
4878            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
4879                self.state.reviews.push(round_record);
4880                return self
4881                    .stop_reviewing(
4882                        "the round's own verification could not run",
4883                        &shell,
4884                        &winner.worktree,
4885                    )
4886                    .await;
4887            }
4888
4889            if round == max_rounds {
4890                self.state.reviews.push(round_record);
4891                return self
4892                    .stop_reviewing(
4893                        &format!(
4894                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
4895                        ),
4896                        &shell,
4897                        &winner.worktree,
4898                    )
4899                    .await;
4900            }
4901
4902            // Fix. The winner's own implementer seat continues its conversation:
4903            // the competition is over, so context is pure benefit now.
4904            let (fix_spec, fix_seat_key) = self.fixer_spec(&winner);
4905            let seat = self.seat(&fix_seat_key, &fix_spec.id);
4906            let blocking_findings: Vec<_> = all_findings
4907                .iter()
4908                .filter(|f| f.severity.blocks())
4909                .cloned()
4910                .collect();
4911            let job = SeatJob {
4912                prompt: prompt::fix(
4913                    &self.state.instruction,
4914                    &blocking_findings,
4915                    this_round_verification.as_ref(),
4916                    round,
4917                    max_rounds,
4918                    &language,
4919                ),
4920                spec: fix_spec.clone(),
4921                seat,
4922                cwd: winner.worktree.clone(),
4923                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
4924                allow_write: true,
4925                sessions,
4926                artifacts: artifacts.clone(),
4927                stem: format!("fix-{round}"),
4928                handover: None,
4929            };
4930            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
4931            let cache = self.state.config.cache_dir();
4932            let ctx = WaveCtx {
4933                carry_seats: false,
4934                run: &run_id,
4935                node: "fix",
4936                prompts: &prompts,
4937                cache: cache.as_deref(),
4938                round: Some(round),
4939            };
4940            let (seat, out) =
4941                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
4942            let agent_id = seat.agent.clone();
4943
4944            let mut fix = FixRecord {
4945                agent: agent_id,
4946                addressed: Vec::new(),
4947                rejected: Vec::new(),
4948                notes: String::new(),
4949                committed: false,
4950                failed: None,
4951                duration_ms: 0,
4952                continuation: None,
4953            };
4954            let mut continuation = ContinuationRecord::not_needed();
4955            let mut final_seat = seat.clone();
4956            match out {
4957                AgentOutcome::Ok(o) => {
4958                    fix.duration_ms = o.duration_ms;
4959                    let parsed = verdict::extract_json::<FixReport>(&o.text);
4960                    // A parsed report standing next to a command this same
4961                    // reply's own CLI never confirmed the exit status of is
4962                    // not a resolved answer — the identical `CommandEvidence`
4963                    // `state.jobs` renders, read here instead of only on
4964                    // display, per the completion judgment and the shown
4965                    // record needing to agree.
4966                    let incomplete_reason = match &parsed {
4967                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
4968                            "the reply parsed, but it reported a command whose own CLI never \
4969                             confirmed an exit status"
4970                                .to_owned(),
4971                        ),
4972                        Ok(_) => None,
4973                        Err(e) => Some(e.to_string()),
4974                    };
4975                    match incomplete_reason {
4976                        None => {
4977                            let report = parsed.expect("checked Ok above");
4978                            fix.addressed = report.addressed;
4979                            fix.rejected = report.rejected;
4980                            fix.notes =
4981                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
4982                        }
4983                        Some(reason) => {
4984                            let (resumed_seat, resolved, failure, cont) = self
4985                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
4986                                .await;
4987                            fix.duration_ms += cont.cumulative_wait_ms;
4988                            continuation = cont;
4989                            final_seat = resumed_seat;
4990                            match resolved {
4991                                Some(report) => {
4992                                    fix.addressed = report.addressed;
4993                                    fix.rejected = report.rejected;
4994                                    fix.notes = blind::sanitize_prose(
4995                                        &report.notes,
4996                                        &self.state.config.blind,
4997                                    );
4998                                }
4999                                None => fix.failed = failure,
5000                            }
5001                        }
5002                    }
5003                }
5004                // The CLI's raw error JSON is not a fix report to parse.
5005                AgentOutcome::Dropped(o) => {
5006                    fix.duration_ms = o.duration_ms;
5007                    let why = o
5008                        .dropped
5009                        .as_ref()
5010                        .map(|d| d.why.as_str())
5011                        .unwrap_or("the CLI ended the stream without delivering its answer");
5012                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
5013                }
5014                AgentOutcome::Quota(o) => {
5015                    self.state.quota.push(QuotaLoss {
5016                        seat: final_seat.key.clone(),
5017                        node: "fix".to_owned(),
5018                        at: Timestamp::now(),
5019                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5020                    });
5021                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
5022                }
5023                AgentOutcome::Failed(e) => fix.failed = Some(e),
5024            }
5025            fix.continuation = Some(continuation);
5026            self.state.seats.insert(final_seat.key.clone(), final_seat);
5027            if let Ok(r) = git::rescue_commit(
5028                &winner.worktree,
5029                &format!("magi: review round {round} fixes (uncommitted work)"),
5030            )
5031            .await
5032            {
5033                self.state.note_withheld("fix", &r.withheld);
5034            }
5035            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5036            fix.committed = after != before;
5037            // Judged by what `git` says moved against base, never by the
5038            // fixer's own `addressed`/`rejected` count — see
5039            // `ReviewRound::progressed`. Propagated with `?`, the same as the
5040            // `patch` snapshot above: swallowing this error would default
5041            // `diff_after` to empty, which almost always differs from a
5042            // non-empty `patch` and reads as "progressed" — exactly backwards
5043            // for a `git` failure the stagnation check cannot see through.
5044            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
5045            let progressed = diff_after != patch;
5046            let commit_note = if fix.committed {
5047                "committed"
5048            } else {
5049                "NO new commit"
5050            };
5051            let tree_note = if progressed {
5052                "changed vs base"
5053            } else {
5054                "unchanged vs base"
5055            };
5056            self.state.event(
5057                "fix",
5058                match &fix.failed {
5059                    // Distinct on purpose from "0 addressed, 0 rejected": the
5060                    // fixer's own diff still landed (blocking counts do keep
5061                    // falling round over round), only its adoption report did
5062                    // not come back, so this must never read like every
5063                    // finding was reviewed and declined.
5064                    Some(reason) => {
5065                        format!(
5066                            "round {round}: fixer's adoption report was lost ({reason}); \
5067                             {commit_note}, tree {tree_note}"
5068                        )
5069                    }
5070                    None => format!(
5071                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
5072                         {tree_note}{}",
5073                        fix.addressed.len(),
5074                        fix.rejected.len(),
5075                        if continuation.outcome == ContinuationOutcome::Resumed {
5076                            format!(
5077                                " (adoption report recovered after {} continuation(s))",
5078                                continuation.attempts
5079                            )
5080                        } else {
5081                            String::new()
5082                        },
5083                    ),
5084                },
5085            );
5086            round_record.fix = Some(fix);
5087            round_record.progressed = progressed;
5088            self.state.reviews.push(round_record);
5089            self.state.save()?;
5090
5091            // The fixer's own report never came back this round, even after
5092            // `continue_fix_report`'s own budget was spent on it — not an
5093            // ordinary "no report" (dropped stream, quota, plain failure),
5094            // which already reads that way and is left to the existing round
5095            // budget. Stopping here, rather than opening another round, is
5096            // what keeps a next reviewer/fixer wave from ever being
5097            // dispatched onto `winner.worktree` while whatever the seat's
5098            // last call may still have running there is unaccounted for: no
5099            // process liveness check exists (and none is being added — see
5100            // AGENTS.md/this task's own scope), so the only way to honour
5101            // "nothing starts before a valid report returns" is to not start
5102            // anything further on this worktree from this run at all.
5103            if matches!(
5104                continuation.outcome,
5105                ContinuationOutcome::Exhausted
5106                    | ContinuationOutcome::QuotaLost
5107                    | ContinuationOutcome::NoSession
5108            ) {
5109                return self
5110                    .stop_reviewing(
5111                        "the fixer's adoption report never came back, even after resuming its \
5112                         own seat; refusing to start another round against the same worktree \
5113                         while that is unresolved",
5114                        &shell,
5115                        &winner.worktree,
5116                    )
5117                    .await;
5118            }
5119
5120            let streak = self
5121                .state
5122                .reviews
5123                .iter()
5124                .rev()
5125                .take_while(|r| !r.progressed)
5126                .count();
5127            if streak >= STAGNANT_LIMIT {
5128                return self
5129                    .stop_reviewing(
5130                        &format!(
5131                            "the tree has not moved against base for {streak} round(s) in a row"
5132                        ),
5133                        &shell,
5134                        &winner.worktree,
5135                    )
5136                    .await;
5137            }
5138        }
5139        Ok(())
5140    }
5141
5142    /// Decide, from the last recorded round's own verification, whether
5143    /// stopping the review loop is a hand-off or a genuine block.
5144    ///
5145    /// Called once the loop has given up trying — the round budget is spent,
5146    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
5147    /// findings still open, never while a round is still clean or the
5148    /// incomplete-panel case handled inline above. Gate and e2e are facts
5149    /// about the tree; a lingering review finding is an opinion, and this
5150    /// workload's own `magi stats` puts reviewer precision low enough
5151    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
5152    /// must not by itself stand between a green, verified change and the
5153    /// human who decides what to do with it. A red e2e is not an opinion, so
5154    /// that case still blocks, with the failing command and a tail of its
5155    /// output recorded here rather than left in `run.json` for someone to go
5156    /// find.
5157    ///
5158    /// A round that deferred its own e2e (see [`Config::graph`]'s
5159    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
5160    /// only because nothing ran, and treating an empty list as a passing one
5161    /// here is exactly the "deferred painted green" bug this function exists
5162    /// to not have. When the last round's own verification never resolved —
5163    /// deferred on purpose, or a real attempt the shared build cache blocked
5164    /// — this makes (or retries) the real run, on the actual worktree this
5165    /// loop is about to stop touching, before deciding anything. A
5166    /// resource-blocked attempt is likewise never read as either green or
5167    /// red: it is evidence about the machine, not the patch (see
5168    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
5169    /// blocked cache leaves this call without deciding rather than guessing
5170    /// — the caller retries on a later reentry.
5171    /// Record, once, that the review loop handed off over a blocking finding
5172    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
5173    /// `land` asks the owner even with `land_approval` off. Called from every
5174    /// path that concludes `Gating`; a reentry keeps the first record.
5175    fn record_contested_handoff(&mut self) {
5176        if self.state.contested_handoff.is_some() {
5177            return;
5178        }
5179        let Some(contested) = self
5180            .state
5181            .reviews
5182            .last()
5183            .and_then(ReviewRound::contested_handoff)
5184        else {
5185            return;
5186        };
5187        self.state.event(
5188            "review",
5189            format!(
5190                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
5191                 wait for the owner's approval",
5192                contested.findings.len(),
5193                contested.rejecters.len()
5194            ),
5195        );
5196        self.state.contested_handoff = Some(contested);
5197    }
5198
5199    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
5200        let round_idx = self.state.reviews.len() - 1;
5201        // A deferred round and a resource-blocked one are the same shape
5202        // here: neither has a real result yet, and both get one more
5203        // attempt. Read off `e2e_status` — the single source for this —
5204        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
5205        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
5206        // still retries instead of being read as a settled result the
5207        // instant it stops being empty.
5208        let needs_catchup_run = matches!(
5209            self.state.reviews[round_idx].e2e_status(),
5210            E2eStatus::Deferred | E2eStatus::ResourceBlocked
5211        );
5212        if needs_catchup_run {
5213            let round = self.state.reviews[round_idx].round;
5214            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5215            let commands = self.state.config.verify.e2e.clone();
5216            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
5217            let cache_dir = self.state.config.cache_dir();
5218            let context = format!(
5219                "round {round}: verification unresolved, catching up before the final decision"
5220            );
5221            let (outcomes, verify_retried) = with_cache_lease(
5222                &mut self.state,
5223                cache_dir.as_deref(),
5224                "e2e",
5225                "e2e",
5226                worktree,
5227                &attempted_head,
5228                timeout,
5229                &context,
5230                |state, budget| {
5231                    let shell = shell.to_vec();
5232                    let commands = commands.clone();
5233                    let context = context.clone();
5234                    async move {
5235                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
5236                            .await
5237                    }
5238                },
5239            )
5240            .await;
5241            let last = &mut self.state.reviews[round_idx];
5242            last.e2e = outcomes;
5243            last.verify_retried = verify_retried;
5244            // Always the commit and time this attempt actually targeted,
5245            // whether or not it happens to equal the reviewed `head` and
5246            // whether or not a command finished — see
5247            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
5248            // attempt is recorded too, so a later reader sees "attempted
5249            // again at T2" rather than silence.
5250            last.verified_head = Some(attempted_head);
5251            last.verified_at = Some(Timestamp::now());
5252            if verify_inconclusive(&last.e2e) {
5253                // Still not a real result: `e2e_deferred` is left exactly
5254                // as it was, so `needs_catchup_run` above reads
5255                // `ResourceBlocked` (via `e2e_status`, which checks
5256                // `resource_blocked` before `e2e_deferred`) and retries
5257                // again on the next reentry, rather than recording
5258                // contention as a red e2e and blocking the run on it.
5259                self.state.save()?;
5260                return Ok(());
5261            }
5262            last.e2e_deferred = false;
5263        }
5264        let last = &self.state.reviews[round_idx];
5265        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
5266
5267        match last.e2e_status() {
5268            E2eStatus::Failed => {
5269                let red: Vec<String> = last
5270                    .e2e
5271                    .iter()
5272                    .filter(|o| !o.ok())
5273                    .map(|o| {
5274                        format!(
5275                            "`{}` -> {:?}\n{}",
5276                            o.command,
5277                            o.code,
5278                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5279                        )
5280                    })
5281                    .collect();
5282                self.state
5283                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
5284                self.state.status = RunStatus::Blocked;
5285            }
5286            // `needs_catchup_run` above already retried once this call; if
5287            // it is still blocked, this is magi's own admission it could
5288            // not get a command to run, never a verdict on the patch — the
5289            // run is left exactly where a later reentry can retry again.
5290            E2eStatus::ResourceBlocked => {
5291                self.state.event(
5292                    "review",
5293                    format!(
5294                        "{why}; e2e could not run (shared build cache unavailable); not \
5295                         deciding yet"
5296                    ),
5297                );
5298            }
5299            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
5300                self.state.event(
5301                    "review",
5302                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
5303                );
5304                self.record_contested_handoff();
5305                self.state.status = RunStatus::Gating;
5306            }
5307        }
5308        self.state.save()?;
5309        Ok(())
5310    }
5311
5312    // ----------------------------------------------------------------- gate
5313
5314    async fn gate(&mut self) -> Result<()> {
5315        // Judged by the review record itself, not by `status`: a solo
5316        // candidate's `judge`/`deliberate` skip rewrites `status` on every
5317        // reentry (see `judge`), and trusting it here is exactly how a run
5318        // that exhausted its review budget got gated and merged a second
5319        // time around. `review_conclusion` recomputes the review loop's own
5320        // verdict from the round records themselves — `Gating` for a clean
5321        // round or a hand-off (see `stop_reviewing`), anything else means the
5322        // loop is still going or genuinely blocked.
5323        // A base the winner could not be replayed onto is a decision, not a
5324        // round: there is no landing tree to gate. Read as its own record for
5325        // the same reason the review verdict is.
5326        if self.state.status == RunStatus::Failed
5327            || self
5328                .state
5329                .base_sync
5330                .as_ref()
5331                .is_some_and(|s| s.conflict.is_some())
5332            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5333                != Some(RunStatus::Gating)
5334        {
5335            return Ok(());
5336        }
5337        if self.state.gate_ran {
5338            // `review_loop` derives its conclusion from the clean review
5339            // record on every reentry and therefore puts a completed run back
5340            // in `Gating`. A recorded gate is a stronger, terminal fact:
5341            // retain its original command output (or lack of any, for a repo
5342            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
5343            // doc) and restore `Blocked` on a real failure rather than
5344            // pretending the command is still running or running it a second
5345            // time. `gate_ran == false` remains the only shape — unattempted,
5346            // or a resource-blocked retry — that may still need to execute a
5347            // command.
5348            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
5349                self.state.status = RunStatus::Blocked;
5350                self.state.save()?;
5351            }
5352            return Ok(());
5353        }
5354        let Some(winner) = self.state.winner().cloned() else {
5355            return Ok(());
5356        };
5357        self.state.status = RunStatus::Gating;
5358        let mut outcomes = self.run_gate(&winner).await?;
5359        loop {
5360            // A resource-blocked outcome means the gate command never actually
5361            // ran - the shared build cache could not be acquired or confirmed
5362            // fresh in time - which is evidence about the machine, not about
5363            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
5364            // Recording it as a red gate would mark a run `Blocked` on nothing
5365            // but contention magi has already logged; leaving `self.state.gate`
5366            // empty and `self.state.gate_ran` false instead keeps the shape
5367            // this function already treats as "still needs to run" (see the
5368            // early-return above), so the next call retries the command
5369            // rather than concluding anything.
5370            if verify_inconclusive(&outcomes) {
5371                self.state.save()?;
5372                return Ok(());
5373            }
5374            if outcomes.iter().all(CommandOutcome::ok) {
5375                break;
5376            }
5377            match self.gate_fix_round(&winner, &outcomes).await? {
5378                GateFix::Retry => outcomes = self.run_gate(&winner).await?,
5379                GateFix::Stop => break,
5380                GateFix::Defer => {
5381                    self.state.save()?;
5382                    return Ok(());
5383                }
5384            }
5385        }
5386        let passed = outcomes.iter().all(CommandOutcome::ok);
5387        self.state.gate = outcomes;
5388        self.state.gate_ran = true;
5389        if !passed {
5390            self.state.status = RunStatus::Blocked;
5391            let spent = self.state.gate_fixes.len();
5392            self.state.event(
5393                "gate",
5394                if spent == 0 {
5395                    "gate failed; not merging".to_owned()
5396                } else {
5397                    format!("gate failed after {spent} gate-fix round(s); not merging")
5398                },
5399            );
5400        }
5401        self.state.save()?;
5402        Ok(())
5403    }
5404
5405    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
5406    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
5407    /// after review is clean and never on a candidate awaiting judging.
5408    ///
5409    /// Never fails the run: a non-zero exit or timeout is a warning and a
5410    /// recorded outcome, and the gate remains the single arbiter. Nothing
5411    /// configured means nothing happens - no event, no commit. `commit_all`
5412    /// commits any leftover change under the neutral identity and returns
5413    /// `false` when the tree is clean, so no empty commit is ever made.
5414    async fn run_pre_gate(&mut self, winner: &Candidate) {
5415        let commands = self.state.config.verify.pre_gate.clone();
5416        if commands.is_empty() {
5417            return;
5418        }
5419        let shell = self.state.config.shell();
5420        let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5421        let (outcomes, _) = run_commands(
5422            &mut self.state,
5423            "pre_gate",
5424            "pre_gate",
5425            0,
5426            &shell,
5427            &commands,
5428            &winner.worktree,
5429            timeout,
5430        )
5431        .await;
5432        for o in &outcomes {
5433            if !o.ok() {
5434                tracing::warn!(
5435                    "pre_gate `{}` failed ({:?}); the gate decides",
5436                    o.command,
5437                    o.code
5438                );
5439            }
5440            self.state.event(
5441                "pre_gate",
5442                format!(
5443                    "`{}` -> {}",
5444                    o.command,
5445                    if o.ok() {
5446                        "pass".to_owned()
5447                    } else {
5448                        format!(
5449                            "FAIL ({:?})\n{}",
5450                            o.code,
5451                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5452                        )
5453                    }
5454                ),
5455            );
5456        }
5457        self.state.pre_gate = outcomes;
5458        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
5459            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
5460                Ok(head) => {
5461                    self.state
5462                        .event("pre_gate", format!("committed mechanical fixes ({head})"));
5463                    self.state.pre_gate_commit = Some(head);
5464                }
5465                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
5466            },
5467            Ok(false) => {}
5468            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
5469        }
5470        if let Err(e) = self.state.save() {
5471            tracing::warn!("could not persist the pre_gate record: {e:#}");
5472        }
5473    }
5474
5475    /// Run `verify.gate` once against the winner's current tree, logging one
5476    /// event per command. Empty when nothing is configured.
5477    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
5478        self.run_pre_gate(winner).await;
5479        let shell = self.state.config.shell();
5480        let gate_commands = self.state.config.verify.gate.clone();
5481        // Zero commands has nothing to run and nothing that could touch the
5482        // shared build cache, so it never needs a lease: `Config::cache_dir`
5483        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
5484        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
5485        // otherwise queue behind an unrelated run's lease and come back
5486        // resource-blocked - `gate_ran` would stay false on nothing but
5487        // cache contention, for a step that had nothing to check in the
5488        // first place.
5489        let outcomes = if gate_commands.is_empty() {
5490            Vec::new()
5491        } else {
5492            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5493            let cache_dir = self.state.config.cache_dir();
5494            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5495            let (outcomes, _) = with_cache_lease(
5496                &mut self.state,
5497                cache_dir.as_deref(),
5498                "gate",
5499                "gate",
5500                &winner.worktree,
5501                &head,
5502                timeout,
5503                "final gate",
5504                |state, budget| {
5505                    let shell = shell.clone();
5506                    let gate_commands = gate_commands.clone();
5507                    let worktree = winner.worktree.clone();
5508                    async move {
5509                        let (outcomes, timed_out_pids) = run_commands(
5510                            state,
5511                            "gate",
5512                            "gate",
5513                            0,
5514                            &shell,
5515                            &gate_commands,
5516                            &worktree,
5517                            budget,
5518                        )
5519                        .await;
5520                        (outcomes, false, timed_out_pids)
5521                    }
5522                },
5523            )
5524            .await;
5525            outcomes
5526        };
5527        if outcomes.is_empty() {
5528            // Nothing configured to check — distinct from every other
5529            // silence in this run's event log, since an empty `gate` alone
5530            // no longer says whether the gate ran at all (see
5531            // `RunState::gate_ran`'s own doc).
5532            self.state.event(
5533                "gate",
5534                "no gate commands configured; nothing to check, passing",
5535            );
5536        }
5537        for o in &outcomes {
5538            self.state.event(
5539                "gate",
5540                format!(
5541                    "`{}` -> {}",
5542                    o.command,
5543                    if o.ok() {
5544                        "pass".to_owned()
5545                    } else {
5546                        format!(
5547                            "FAIL ({:?})\n{}",
5548                            o.code,
5549                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
5550                        )
5551                    }
5552                ),
5553            );
5554        }
5555        Ok(outcomes)
5556    }
5557
5558    /// One bounded fix round for a failing gate.
5559    ///
5560    /// The fixer is told the failure came from the gate itself, not from a
5561    /// reviewer, and is shown the failed commands, their exit codes and a tail
5562    /// of their output - whatever `[verify].gate` holds, nothing here knows
5563    /// what those commands run. Only a normal non-zero exit that printed
5564    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
5565    /// command or a full disk says nothing about the code, and a fixer sent
5566    /// after it can only appease the machine. The round is judged by what git
5567    /// says moved, never by the fixer's own report, and `verify.e2e` runs
5568    /// again before the gate does, so a fix cannot trade a green gate for a
5569    /// red e2e unnoticed.
5570    async fn gate_fix_round(
5571        &mut self,
5572        winner: &Candidate,
5573        outcomes: &[CommandOutcome],
5574    ) -> Result<GateFix> {
5575        let cap = self.state.config.graph.gate_fix_rounds;
5576        let spent = self.state.gate_fixes.len();
5577        if spent >= cap {
5578            if cap > 0 {
5579                self.state.event(
5580                    "gate",
5581                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
5582                );
5583            }
5584            return Ok(GateFix::Stop);
5585        }
5586        if !gate_fixable(outcomes) {
5587            self.state.event(
5588                "gate",
5589                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
5590                 command or similar); not spending a fix round on it",
5591            );
5592            return Ok(GateFix::Stop);
5593        }
5594        let min_free = self.state.config.disk.min_free_bytes;
5595        if min_free > 0 {
5596            match crate::disk::free_bytes(&winner.worktree) {
5597                Ok(free) if crate::disk::enough_space(free, min_free) => {}
5598                Ok(free) => {
5599                    self.state.event(
5600                        "gate",
5601                        format!(
5602                            "only {free} bytes free ({min_free} required by `[disk] \
5603                             min_free_bytes`); not spending a fix round on a failure the disk \
5604                             may explain"
5605                        ),
5606                    );
5607                    return Ok(GateFix::Stop);
5608                }
5609                Err(e) => {
5610                    self.state.event(
5611                        "gate",
5612                        format!("free disk space could not be measured ({e:#}); no fix round"),
5613                    );
5614                    return Ok(GateFix::Stop);
5615                }
5616            }
5617        }
5618
5619        let attempt = spent + 1;
5620        let run_id = self.state.id.clone();
5621        let prompts = self.state.config.prompts.clone();
5622        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
5623        let base = self.landing_base();
5624        let (fix_spec, fix_seat_key) = self.fixer_spec(winner);
5625        let seat = self.seat(&fix_seat_key, &fix_spec.id);
5626        let job = SeatJob {
5627            prompt: prompt::gate_fix(
5628                &self.state.instruction,
5629                &failed,
5630                attempt,
5631                cap,
5632                &self.state.config.graph.language,
5633            ),
5634            spec: fix_spec,
5635            seat,
5636            cwd: winner.worktree.clone(),
5637            timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
5638            allow_write: true,
5639            sessions: self.state.config.graph.sessions,
5640            artifacts: agent::artifacts_dir(&self.state.dir()),
5641            stem: format!("gate-fix-{attempt}"),
5642            handover: None,
5643        };
5644        self.state.event(
5645            "gate",
5646            format!("gate failed; gate-fix round {attempt} of {cap}"),
5647        );
5648        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
5649        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
5650        let cache = self.state.config.cache_dir();
5651        let ctx = WaveCtx {
5652            carry_seats: false,
5653            run: &run_id,
5654            node: "gate-fix",
5655            prompts: &prompts,
5656            cache: cache.as_deref(),
5657            round: None,
5658        };
5659        let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
5660        let mut record = GateFixRecord {
5661            agent: seat.agent.clone(),
5662            failed,
5663            notes: String::new(),
5664            committed: false,
5665            error: None,
5666        };
5667        match out {
5668            AgentOutcome::Ok(o) => {
5669                // A missing report is not a failed fix: the round is judged
5670                // by the tree below, and the report only carries prose.
5671                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
5672                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
5673                }
5674            }
5675            AgentOutcome::Dropped(_) => {
5676                record.error = Some("the CLI dropped the stream".to_owned());
5677            }
5678            AgentOutcome::Quota(o) => {
5679                self.state.quota.push(QuotaLoss {
5680                    seat: seat.key.clone(),
5681                    node: "gate-fix".to_owned(),
5682                    at: Timestamp::now(),
5683                    reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
5684                });
5685                record.error = Some("rate limited (quota); fixer could not run".to_owned());
5686            }
5687            AgentOutcome::Failed(e) => record.error = Some(e),
5688        }
5689        self.state.seats.insert(seat.key.clone(), seat);
5690        if let Ok(r) = git::rescue_commit(
5691            &winner.worktree,
5692            &format!("magi: gate fix {attempt} (uncommitted work)"),
5693        )
5694        .await
5695        {
5696            self.state.note_withheld("gate-fix", &r.withheld);
5697        }
5698        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
5699        record.committed = after != before;
5700        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
5701        let note = record.error.clone();
5702        self.state.gate_fixes.push(record);
5703        self.state.save()?;
5704        if !changed {
5705            self.state.event(
5706                "gate",
5707                match note {
5708                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
5709                    None => format!("gate-fix round {attempt}: the tree did not change"),
5710                },
5711            );
5712            return Ok(GateFix::Stop);
5713        }
5714        self.state.event(
5715            "gate",
5716            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
5717        );
5718
5719        let commands = self.state.config.verify.e2e.clone();
5720        if !commands.is_empty() {
5721            let shell = self.state.config.shell();
5722            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
5723            let cache_dir = self.state.config.cache_dir();
5724            let context = format!("gate-fix round {attempt}");
5725            let (e2e, _) = with_cache_lease(
5726                &mut self.state,
5727                cache_dir.as_deref(),
5728                "e2e",
5729                "e2e",
5730                &winner.worktree,
5731                &after,
5732                timeout,
5733                &context,
5734                |state, budget| {
5735                    let shell = shell.clone();
5736                    let commands = commands.clone();
5737                    let context = context.clone();
5738                    let worktree = winner.worktree.clone();
5739                    async move {
5740                        run_e2e_with_retry(state, &shell, &commands, &worktree, budget, &context)
5741                            .await
5742                    }
5743                },
5744            )
5745            .await;
5746            if verify_inconclusive(&e2e) {
5747                return Ok(GateFix::Defer);
5748            }
5749            if e2e.iter().any(|o| !o.ok()) {
5750                self.state.event(
5751                    "gate",
5752                    format!("gate-fix round {attempt}: verify.e2e failed after the fix"),
5753                );
5754                return Ok(GateFix::Stop);
5755            }
5756        }
5757        Ok(GateFix::Retry)
5758    }
5759
5760    // ---------------------------------------------------------------- merge
5761
5762    async fn merge(&mut self) -> Result<()> {
5763        // Same reasoning as `gate`: ask the review and gate records directly
5764        // rather than `status`, which a solo-candidate `judge`/`deliberate`
5765        // skip can rewrite on reentry to something that no longer says
5766        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
5767        // so a hand-off (open findings, green verification) reaches merge
5768        // exactly like a genuinely clean round does.
5769        //
5770        // A run resumed mid-`land` never reaches here at all: `execute`
5771        // recognises `RunStatus::Landing` before it even calls `prep`, and
5772        // routes straight to `run_land` instead. That has to happen a level
5773        // up from this function, not with a check in here, because
5774        // `review_loop`'s own status recomputation (see its doc) runs
5775        // *before* `merge` on every reentry and would otherwise overwrite
5776        // the `Landing` marker with `Gating` before this node ever saw it.
5777        if self
5778            .state
5779            .base_sync
5780            .as_ref()
5781            .is_some_and(|s| s.conflict.is_some())
5782            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
5783                != Some(RunStatus::Gating)
5784            // `gate_ran == false` is not "passed" - `gate` leaves it false
5785            // both before it has ever run and when its last attempt was
5786            // resource-blocked (see `Runner::gate`'s own doc), and neither is
5787            // permission to merge on nothing but the review record. Only a
5788            // gate that actually ran - zero commands configured and
5789            // vacuously passed, or one or more that all exited 0 - may
5790            // proceed; `RunState::gate_status` is the single place that
5791            // reading is computed.
5792            || !self.state.gate_status().ok()
5793        {
5794            return Ok(());
5795        }
5796        // This node's own record, not `status`: `status == Ready` is not
5797        // unique to the harmless `MergeMode::None` path this line was
5798        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
5799        // run's PR was closed without merging — and on that run `mode` is
5800        // still `Pr`, so a reentry that fell through here would push and
5801        // open a second pull request. `self.state.merge` is set exactly once
5802        // this node (or `land`) has already produced a verdict, under every
5803        // mode, which is what "already done" actually means here.
5804        if self.state.merge.is_some() {
5805            return Ok(());
5806        }
5807        let Some(winner) = self.state.winner().cloned() else {
5808            return Ok(());
5809        };
5810        let repo = self.state.repo.clone();
5811        let base = self.state.base_branch.clone();
5812        let mode = self.state.config.merge.mode;
5813        let style = self.state.config.merge.style;
5814        let facts = if is_review_run(&self.state) {
5815            branch_facts(&repo, &self.state.base_commit, &winner.branch).await
5816        } else {
5817            None
5818        };
5819        let pr = pr_message_with(&self.state, winner.label, facts.as_ref());
5820        let message = pr.commit_message();
5821
5822        let outcome = match mode {
5823            MergeMode::None => MergeOutcome {
5824                mode,
5825                ok: true,
5826                detail: manual_merge_command(style, &repo, &winner.branch, &message),
5827                empty: false,
5828            },
5829            MergeMode::Pr | MergeMode::Local
5830                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
5831            {
5832                MergeOutcome {
5833                    mode,
5834                    ok: false,
5835                    detail: empty_candidate_detail(&self.state, &base),
5836                    empty: true,
5837                }
5838            }
5839            MergeMode::Local => {
5840                let on = git::current_branch(&repo).await?;
5841                if on.as_deref() != Some(base.as_str()) {
5842                    MergeOutcome {
5843                        mode,
5844                        ok: false,
5845                        detail: format!(
5846                            "{} has {} checked out, not the base branch {base}",
5847                            repo.display(),
5848                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
5849                        ),
5850                        empty: false,
5851                    }
5852                } else if !git::is_clean(&repo).await? {
5853                    MergeOutcome {
5854                        mode,
5855                        ok: false,
5856                        detail: format!("{} is dirty; refusing to merge", repo.display()),
5857                        empty: false,
5858                    }
5859                } else {
5860                    let out = match style {
5861                        MergeStyle::Merge => {
5862                            git::merge_no_ff(&repo, &winner.branch, &message).await?
5863                        }
5864                        MergeStyle::Squash => {
5865                            git::merge_squash(&repo, &winner.branch, &message).await?
5866                        }
5867                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
5868                    };
5869                    MergeOutcome {
5870                        mode,
5871                        ok: out.ok(),
5872                        detail: if out.ok() { out.stdout } else { out.stderr },
5873                        empty: false,
5874                    }
5875                }
5876            }
5877            MergeMode::Pr => {
5878                let remote = self.state.config.merge.remote.clone();
5879                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
5880                if !pushed.ok() {
5881                    MergeOutcome {
5882                        mode,
5883                        ok: false,
5884                        detail: pushed.stderr,
5885                        empty: false,
5886                    }
5887                } else {
5888                    // A retry or resume of a run whose branch already has an
5889                    // open pull request adopts it rather than failing on a
5890                    // duplicate. Only this winner branch into this base:
5891                    // `branch_for` derives the name from the run id, so a
5892                    // different run's pull request never matches.
5893                    let found = land::find_open_pr(&winner.worktree, &winner.branch, &base).await;
5894                    let out = match pr_merge_plan(found) {
5895                        PrPlan::Create => {
5896                            gh_pr_create(
5897                                &winner.worktree,
5898                                &base,
5899                                &winner.branch,
5900                                &pr.title,
5901                                &pr.body,
5902                            )
5903                            .await
5904                        }
5905                        PrPlan::Adopt { url, title } => {
5906                            self.state
5907                                .event("merge", format!("Pr: adopted open pull request {url}"));
5908                            if title != pr.title
5909                                && let Err(e) =
5910                                    land::set_pr_title(&winner.worktree, &url, &pr.title).await
5911                            {
5912                                tracing::warn!("could not refresh title of {url}: {e:#}");
5913                                self.state
5914                                    .event("merge", format!("Pr: title refresh failed: {e:#}"));
5915                            }
5916                            Ok(url)
5917                        }
5918                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
5919                    };
5920                    match out {
5921                        Ok(url) => MergeOutcome {
5922                            mode,
5923                            ok: true,
5924                            detail: url,
5925                            empty: false,
5926                        },
5927                        Err(e) => MergeOutcome {
5928                            mode,
5929                            ok: false,
5930                            detail: e.to_string(),
5931                            empty: false,
5932                        },
5933                    }
5934                }
5935            }
5936        };
5937
5938        self.state.status = match (mode, outcome.ok) {
5939            (MergeMode::None, _) => RunStatus::Ready,
5940            (_, true) => RunStatus::Merged,
5941            (_, false) => RunStatus::Blocked,
5942        };
5943        self.state.event(
5944            "merge",
5945            format!(
5946                "{:?}: {}",
5947                mode,
5948                outcome.detail.lines().next().unwrap_or("")
5949            ),
5950        );
5951        self.state.merge = Some(outcome);
5952        self.state.save()?;
5953
5954        // The PR is open and the run would historically stop here, leaving the
5955        // operator to watch checks, feed review comments back to a fixer, and
5956        // merge. That was done by hand six times in one session before this
5957        // existed. Opt-in, because merging is the one irreversible thing magi
5958        // can do to a repository.
5959        if self.state.config.graph.land
5960            && mode == MergeMode::Pr
5961            && self.state.status == RunStatus::Merged
5962        {
5963            self.run_land().await?;
5964        }
5965        // `run_land` may have left `status` at `Landing` - still waiting on
5966        // CI or the owner's approval, not actually settled - so this has to
5967        // read whatever `status` ended up as here, not the `Merged` this
5968        // function set a few lines up.
5969        self.settle_questions();
5970        Ok(())
5971    }
5972
5973    /// Enter `land`.
5974    ///
5975    /// Shared between a fresh run's first pass through [`Runner::merge`] and
5976    /// a resumed run's re-entry. `land::land` itself is what serialises the
5977    /// two git-mutating moments inside the loop — the rebase push and
5978    /// `gh pr merge` — per repository (see its own doc); nothing here needs
5979    /// to hold a lock across the whole call, and doing so would serialise
5980    /// this run's CI wait against a *different* run's land-approval resume
5981    /// in the same repository, which is exactly the "must not wait on
5982    /// another task" property the daemon's slot-freeing exists to give.
5983    async fn run_land(&mut self) -> Result<()> {
5984        let url = self
5985            .state
5986            .merge
5987            .as_ref()
5988            .map(|m| m.detail.clone())
5989            .unwrap_or_default();
5990        let url = url.lines().next().unwrap_or("").trim().to_owned();
5991        if !url.starts_with("http") {
5992            return Ok(());
5993        }
5994        // A land failure is not a lost run: the work is on a branch and the
5995        // pull request is open, which is exactly where a human takes over.
5996        match land::land(&mut self.state, &url).await {
5997            Ok(pr) if self.state.parked => {
5998                // `land` already saved the parked marker; nothing here
5999                // overrides `status` back to a terminal value while an
6000                // approval is still outstanding.
6001                let _ = pr;
6002            }
6003            Ok(pr) => {
6004                self.state.status = match pr.state {
6005                    land::PrLifecycle::Merged => RunStatus::Merged,
6006                    _ => RunStatus::Blocked,
6007                };
6008                // Downstream of a confirmed merge only - see
6009                // `bump::should_release_bump`'s own doc for why this one
6010                // check covers all three of `land`'s success paths.
6011                // Best-effort: the run already landed, so a failure here
6012                // (the decision call, `gh`, `cargo`) is recorded and never
6013                // turns a landed run into a failed one.
6014                if bump::should_release_bump(self.state.status)
6015                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
6016                {
6017                    // The event is the run's own record. Not-eligible cases
6018                    // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
6019                    // `Err` is a bump that was tried and failed:
6020                    // `after_merge` itself raises the operator notice for
6021                    // that, whether or not a release PR exists yet.
6022                    self.state
6023                        .event("bump", format!("release bump skipped: {e:#}"));
6024                }
6025                // Independent of the bump, and best-effort in the same way:
6026                // findings the merge left open become follow-up tasks.
6027                if self.state.status == RunStatus::Merged {
6028                    crate::followup::after_merge(&mut self.state, &pr.url).await;
6029                }
6030                self.state.save()?;
6031            }
6032            Err(e) => {
6033                self.state.status = RunStatus::Blocked;
6034                self.state.event("land", format!("gave up: {e}"));
6035                self.state.save()?;
6036            }
6037        }
6038        Ok(())
6039    }
6040
6041    // -------------------------------------------------------------- helpers
6042
6043    /// Fetch or create a seat, keeping its conversation across nodes.
6044    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
6045        if let Some(existing) = self.state.seats.get(key)
6046            && existing.agent == agent
6047        {
6048            return existing.clone();
6049        }
6050        // A seat that changes agent mints its session id from the agent too,
6051        // like a handover: the old agent's uuid is already taken by the CLI.
6052        let fresh = if self.state.seats.contains_key(key) {
6053            handover_seat(key, agent, self.state.next_seat_seed())
6054        } else {
6055            SeatState::new(key, agent, self.state.seed)
6056        };
6057        self.state.seats.insert(key.to_owned(), fresh.clone());
6058        fresh
6059    }
6060
6061    /// The agent now holding seat `key`: `spec`, unless a handover moved the
6062    /// seat to another roster agent, in which case that agent. Nodes that
6063    /// continue a seat's conversation (deliberation, the votes, a reviewer's
6064    /// reconsideration) must keep talking to whoever answered it, not slip
6065    /// back to the agent that failed it.
6066    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
6067        match self.state.seats.get(key) {
6068            Some(s) if s.agent != spec.id => {
6069                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
6070            }
6071            _ => spec,
6072        }
6073    }
6074
6075    /// A candidate rendered for judging, with the leak policy applied.
6076    fn view(&self, c: &Candidate) -> CandidateView {
6077        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
6078            .unwrap_or_default();
6079        let (patch, _) = blind::sanitize_patch(
6080            &format!("candidate {} patch", c.label),
6081            &raw,
6082            &self.state.config.blind,
6083        );
6084        CandidateView {
6085            label: c.label,
6086            branch: c.branch.clone(),
6087            summary: c.summary.clone(),
6088            stat: c.stat.clone(),
6089            patch,
6090        }
6091    }
6092
6093    /// The full candidate set as prompt text, for seats with no live session.
6094    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
6095        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
6096        prompt::judge(
6097            "(see above)",
6098            &views,
6099            self.roles.judges.len(),
6100            base_short,
6101            "en",
6102        )
6103    }
6104
6105    /// The final-vote prompt with everything a seat that has no session of its
6106    /// own needs: the candidates, the seat's own ranking and reasons, and the
6107    /// anonymised deliberation it took part in (only when there was one, so a
6108    /// handed-over seat never sees more than the seat it replaces did). The
6109    /// `Final vote` heading stays first.
6110    fn vote_prompt_full(
6111        &self,
6112        j: usize,
6113        labels: &[char],
6114        language: &str,
6115        candidates: &[Candidate],
6116        base_short: &str,
6117    ) -> String {
6118        let mut text = format!(
6119            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
6120            prompt::final_vote(labels, language),
6121            self.state.instruction,
6122            self.candidate_block(candidates, base_short)
6123        );
6124        if let Some(own) = self
6125            .state
6126            .judgements
6127            .get(j)
6128            .filter(|r| !r.ranking.is_empty())
6129        {
6130            let reasons = own
6131                .reasons
6132                .iter()
6133                .map(|(k, v)| format!("- {k}: {v}"))
6134                .collect::<Vec<_>>()
6135                .join("\n");
6136            text.push_str(&format!(
6137                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
6138                own.ranking.iter().collect::<String>(),
6139                if reasons.is_empty() {
6140                    ""
6141                } else {
6142                    ", because:\n"
6143                }
6144            ));
6145        }
6146        if !self.state.deliberation.is_empty() {
6147            text.push_str("\n# What was argued before this vote\n");
6148            for t in self.transcript(&[], j) {
6149                text.push_str(&format!(
6150                    "\n## {}{}\n\n{}\n",
6151                    t.who,
6152                    if t.is_self { " (you)" } else { "" },
6153                    t.body.trim()
6154                ));
6155            }
6156        }
6157        text
6158    }
6159
6160    /// Anonymised transcript for judge `self_idx`.
6161    ///
6162    /// The initial rankings are always the opening statements. Seeding them
6163    /// only when no turn had been taken yet meant every judge after the first
6164    /// argued against a single voice instead of against the actual split — the
6165    /// disagreement is the information, so it is always on the table.
6166    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
6167        let mut turns = Vec::new();
6168        for j in &self.state.judgements {
6169            if j.ranking.is_empty() {
6170                continue;
6171            }
6172            let reasons = j
6173                .reasons
6174                .iter()
6175                .map(|(k, v)| format!("- {k}: {v}"))
6176                .collect::<Vec<_>>()
6177                .join("\n");
6178            turns.push(Turn {
6179                who: format!("Judge {} (opening ranking)", j.judge),
6180                is_self: j.judge == self_idx + 1,
6181                body: format!(
6182                    "Ranked {}{}{reasons}",
6183                    j.ranking.iter().collect::<String>(),
6184                    if reasons.is_empty() {
6185                        ""
6186                    } else {
6187                        ", because:\n"
6188                    }
6189                ),
6190            });
6191        }
6192        for t in self
6193            .state
6194            .deliberation
6195            .iter()
6196            .flat_map(|r| r.turns.iter())
6197            .chain(current)
6198        {
6199            turns.push(Turn {
6200                who: format!("Judge {}", t.judge),
6201                is_self: t.judge == self_idx + 1,
6202                body: t.body.clone(),
6203            });
6204        }
6205        turns
6206    }
6207}
6208
6209/// Does this seat still hold the context a follow-up prompt would rely on?
6210fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
6211    agent::has_session(spec.kind, seat, sessions)
6212}
6213
6214/// The next entry in `roster` after `start`, never wrapping back to the
6215/// front, whose id is not in `tried` yet.
6216///
6217/// Starts one past `start` rather than at the front of `roster`: `start` is
6218/// the seat's own original position, and a seat whose candidate slot already
6219/// sits on the roster's second entry must fall through to the third next, not
6220/// restart at the first — which is very likely a different candidate's own
6221/// agent already. Never wraps back past `start`, for the same reason: an
6222/// entry earlier in the roster than the seat's own position is almost
6223/// certainly some *other* candidate slot's own agent, and once the tail of
6224/// the roster is exhausted there are no more untried agents for *this* seat
6225/// to fall through to — the caller's fallback chain ends there, exactly as
6226/// "no further untried agents remain in the list for that seat" asks for.
6227///
6228/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
6229/// the same id twice (an operator's `roles.implementers` typo, or a
6230/// `[[agents]]` list reused across roles) must not let
6231/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
6232/// over `roster` either finds an untried id or runs out, so this always
6233/// terminates regardless of duplicates.
6234fn next_untried_in_roster<'a>(
6235    roster: &'a [AgentSpec],
6236    start: usize,
6237    tried: &BTreeSet<String>,
6238) -> Option<&'a AgentSpec> {
6239    roster
6240        .get(start + 1..)?
6241        .iter()
6242        .find(|s| !tried.contains(&s.id))
6243}
6244
6245/// The next agent for a seat that carries its failure history across rounds
6246/// (the review loop). `round_tried` is this round's own bound and starts
6247/// empty every round; `carried_failed` only decides priority.
6248///
6249/// First: an id walking forward from `start`, never wrapping, that is neither
6250/// tried this round nor failed in an earlier one. Only when that is exhausted
6251/// does it rescue: the first roster id (in roster order, so this one *does*
6252/// look before `start`) not yet tried this round, which is by then a carried
6253/// failure. Each id is rescued at most once per round, so it cannot loop.
6254fn next_for_seat<'a>(
6255    roster: &'a [AgentSpec],
6256    start: usize,
6257    round_tried: &BTreeSet<String>,
6258    carried_failed: &BTreeSet<String>,
6259) -> Option<&'a AgentSpec> {
6260    roster
6261        .get(start + 1..)?
6262        .iter()
6263        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
6264        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
6265}
6266
6267/// Where a reviewer seat starts a round: the agent that last answered it when
6268/// it is still on the roster and not marked failed, else the spec's own agent
6269/// unless it failed, else the next roster agent that has not failed, else the
6270/// spec's own agent again (the whole roster failed). Ids no longer on the
6271/// roster are ignored. An empty roster has no handover, so the spec stands.
6272fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
6273    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
6274        return spec;
6275    };
6276    let ok = |id: &str| !h.failed.contains(id);
6277    if let Some(last) = h.last_ok.as_deref()
6278        && ok(last)
6279        && let Some(s) = roster.iter().find(|s| s.id == last)
6280    {
6281        return s.clone();
6282    }
6283    if ok(&spec.id) {
6284        return spec;
6285    }
6286    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
6287    roster
6288        .get(from + 1..)
6289        .into_iter()
6290        .flatten()
6291        .chain(roster.iter())
6292        .find(|s| ok(&s.id))
6293        .cloned()
6294        .unwrap_or(spec)
6295}
6296
6297/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
6298/// seed so a CLI that mints its session id up front (`--session-id`) never
6299/// reuses the uuid the previous agent already opened under the same seat key.
6300fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
6301    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
6302}
6303
6304/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
6305/// so the classifier below and the code that builds the message cannot drift.
6306const TIMED_OUT: &str = "timed out";
6307
6308impl FailClass {
6309    /// `None` for an answer; otherwise how the turn failed.
6310    fn of(out: &AgentOutcome) -> Option<Self> {
6311        match out {
6312            AgentOutcome::Ok(_) => None,
6313            AgentOutcome::Quota(_) => Some(Self::Quota),
6314            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
6315            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
6316            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
6317        }
6318    }
6319
6320    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
6321    fn stem_word(&self) -> &'static str {
6322        match self {
6323            Self::Quota => "quota",
6324            _ => "handover",
6325        }
6326    }
6327}
6328
6329/// The message's first line with its variable parts removed — digit runs and
6330/// path-like tokens — so "exited with Some(2)" and "exited with Some(7)" read
6331/// as one kind of failure.
6332fn failure_signature(msg: &str) -> String {
6333    let line = msg.lines().next().unwrap_or("").trim().to_lowercase();
6334    let mut out = Vec::new();
6335    for word in line.split_whitespace() {
6336        if word.contains('/') || word.contains('\\') {
6337            out.push("<path>".to_owned());
6338            continue;
6339        }
6340        let mut w = String::new();
6341        let mut in_digits = false;
6342        for c in word.chars() {
6343            if c.is_ascii_digit() {
6344                if !in_digits {
6345                    w.push('#');
6346                }
6347                in_digits = true;
6348            } else {
6349                in_digits = false;
6350                w.push(c);
6351            }
6352        }
6353        out.push(w);
6354    }
6355    out.join(" ").chars().take(120).collect()
6356}
6357
6358/// Whether a seat that just failed with `cur` may go to the next roster agent.
6359/// A quota or a timeout always may. Any other failure may not when the agent
6360/// before it failed the same way: an error the prompt causes would otherwise
6361/// walk the whole roster. `prev` is the class of the immediately preceding
6362/// agent's failure, so a quota or timeout in between breaks the run of
6363/// identical failures by itself.
6364fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
6365    match cur {
6366        FailClass::Quota | FailClass::Timeout => true,
6367        FailClass::Other(_) => prev != Some(cur),
6368    }
6369}
6370
6371/// A short human reason for a failed outcome, for the handover record.
6372fn fail_reason(out: &AgentOutcome) -> String {
6373    match out {
6374        AgentOutcome::Ok(_) => String::new(),
6375        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
6376        AgentOutcome::Dropped(o) => format!(
6377            "the CLI dropped the stream ({})",
6378            o.dropped
6379                .as_ref()
6380                .map(|d| d.why.as_str())
6381                .unwrap_or("it ended without delivering its answer")
6382        ),
6383        AgentOutcome::Failed(e) => e.lines().next().unwrap_or("").chars().take(160).collect(),
6384    }
6385}
6386
6387/// Note one handover in the run: the structured record and, in the timeline,
6388/// the sentence a person reads. A quota keeps the wording it always had.
6389fn record_handover(
6390    state: &mut RunState,
6391    node: &str,
6392    seat: &str,
6393    from: &str,
6394    to: &str,
6395    class: &FailClass,
6396    reason: &str,
6397) {
6398    let message = if *class == FailClass::Quota {
6399        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
6400    } else {
6401        format!("{seat}: handed over {from} -> {to} ({reason})")
6402    };
6403    state.event(node, message);
6404    state.handovers.push(Handover {
6405        at: Timestamp::now(),
6406        node: node.to_owned(),
6407        seat: seat.to_owned(),
6408        from: from.to_owned(),
6409        to: to.to_owned(),
6410        reason: reason.to_owned(),
6411    });
6412}
6413
6414/// Did this reply report running a command whose own CLI never confirmed an
6415/// exit status?
6416///
6417/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
6418/// command *finished* (see that type's own doc), so this can only be `true`
6419/// for a command whose completion event carried no readable exit code — not
6420/// for one that simply is not mentioned at all. That is the one signal this
6421/// crate can read, from the same record `state.jobs` renders, about a reply
6422/// standing next to work its own CLI cannot vouch for finishing; it is
6423/// deliberately not a check on the exit code's *value* (a fixer legitimately
6424/// runs a command that fails mid-iteration before it succeeds) and not a
6425/// guess at a command still running in the background (which emits no event
6426/// at all, and so leaves no evidence here to find).
6427fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
6428    commands.iter().any(|c| c.exit_code.is_none())
6429}
6430
6431/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
6432/// trusted as a verified no-op — the adoption guard's own text-level half.
6433///
6434/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
6435/// timed out): a marker only earns the benefit of the doubt from a turn the
6436/// CLI itself vouches for finishing properly, the same house style
6437/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
6438/// *fix* report to for `commands`. A candidate that timed out, exited
6439/// non-zero, or left a command unconfirmed is read as the ordinary loss it
6440/// is, whatever prose it wrote — this returns `None` before it ever looks at
6441/// `text`. The remaining guards (the tree really is empty, the evidence is
6442/// non-empty) are the caller's: this only reads what the reply *claimed*.
6443fn verified_noop_claim(
6444    usable: bool,
6445    commands: &[agent::CommandEvidence],
6446    text: &str,
6447) -> Option<String> {
6448    (usable && !has_unconfirmed_command(commands))
6449        .then(|| verdict::verified_noop(text))
6450        .flatten()
6451}
6452
6453fn short(commit: &str) -> String {
6454    commit.chars().take(7).collect()
6455}
6456
6457fn make_executable(path: &Path) -> Result<()> {
6458    #[cfg(unix)]
6459    {
6460        use std::os::unix::fs::PermissionsExt as _;
6461        let mut perms = std::fs::metadata(path)?.permissions();
6462        perms.set_mode(0o755);
6463        std::fs::set_permissions(path, perms)?;
6464    }
6465    #[cfg(not(unix))]
6466    {
6467        let _ = path;
6468    }
6469    Ok(())
6470}
6471
6472/// What every seat in one batch shares: where the answers are attributed, the
6473/// prompt overlay they inherit, and the build cache they are told to use.
6474///
6475/// A struct rather than four more parameters: `wave` also needs the run's
6476/// state (to record who is answering right now) and the attempt number, and
6477/// eight positional arguments is both unreadable and a clippy error.
6478struct WaveCtx<'a> {
6479    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
6480    /// paid for it.
6481    run: &'a str,
6482    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
6483    node: &'a str,
6484    prompts: &'a Prompts,
6485    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
6486    cache: Option<&'a Path>,
6487    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
6488    /// `JobRecord::round`. `None` for every other node.
6489    round: Option<usize>,
6490    /// Carry each seat's failed-agent history across waves (the review loop
6491    /// only): start-of-round priority and handover choice read
6492    /// [`RunState::seat_history`], and every answer or failure writes it.
6493    carry_seats: bool,
6494}
6495
6496/// Run one job, honouring the parallelism budget.
6497async fn run_one(
6498    job: SeatJob,
6499    sem: Arc<Semaphore>,
6500    ctx: &WaveCtx<'_>,
6501    state: &mut RunState,
6502    attempt: usize,
6503) -> (SeatState, AgentOutcome) {
6504    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
6505        .await
6506        .pop()
6507        .expect("one job in, one result out");
6508    (seat, out)
6509}
6510
6511/// Run every job concurrently, capped by the semaphore, preserving order.
6512///
6513/// Every seat in the batch is recorded into [`RunState::active`] before the
6514/// wave starts and cleared as each answer lands, so the run's own record says
6515/// who is still being waited on rather than only who finished.
6516async fn wave(
6517    jobs: Vec<SeatJob>,
6518    sem: Arc<Semaphore>,
6519    ctx: &WaveCtx<'_>,
6520    state: &mut RunState,
6521    attempt: usize,
6522) -> Vec<(usize, SeatState, AgentOutcome)> {
6523    let WaveCtx {
6524        run,
6525        node,
6526        prompts,
6527        cache,
6528        round,
6529        carry_seats: _,
6530    } = *ctx;
6531    for job in &jobs {
6532        state.seat_started(node, &job.seat.key, job.timeout, attempt);
6533    }
6534    if let Err(e) = state.save() {
6535        // A failed persist of "who is answering right now" must not abort the
6536        // wave: the seats are already being asked, and the alternative is
6537        // losing the answers to save a status line nobody may even be
6538        // watching.
6539        tracing::warn!("could not persist in-progress seats: {e:#}");
6540    }
6541    // Hold the shared build cache's lease for the whole batch, not per job:
6542    // several candidates (an implement wave) or a fixer legitimately share
6543    // one cache concurrently within this run, and that stays untouched — a
6544    // single lease taken once for the whole wave and released once it is
6545    // done is what stops a *different* borrower (another run's own wave, its
6546    // e2e/gate, a human's `magi review`) from interleaving a build into the
6547    // same directory while this one is in flight. Best-effort, not
6548    // all-or-nothing: a wave that cannot get the lease within its own
6549    // longest job's budget still runs — an hour of paid implementer calls is
6550    // not thrown away over cache contention — but every write-allowed seat
6551    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
6552    // below), the same fallback a read-only seat always gets, rather than
6553    // building into a directory this run was never granted. The identity
6554    // record is still invalidated below either way, so the next tracked
6555    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
6556    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
6557    let wait_started = Instant::now();
6558    let cache_guard = if let Some(cache_dir) = cache {
6559        if jobs_had_a_writer {
6560            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
6561            let budget = jobs
6562                .iter()
6563                .map(|j| j.timeout)
6564                .max()
6565                .unwrap_or(Duration::from_secs(60));
6566            acquire_cache_lease(state, cache_dir, &owner, budget, node)
6567                .await
6568                .ok()
6569        } else {
6570            None
6571        }
6572    } else {
6573        None
6574    };
6575    // Carved out of each job's own budget, not added on top of it: a seat
6576    // that waited behind the lease must not also get its full timeout
6577    // afterward, or a run contended on the cache could double the time it
6578    // spends per wave. `saturating_sub` floors at zero rather than
6579    // wrapping - a job whose whole budget was spent waiting starts with
6580    // none left, which is the honest number, not a free minimum.
6581    let waited_for_lease = wait_started.elapsed();
6582    let mut set = tokio::task::JoinSet::new();
6583    let overlay = prompts.overlay(node);
6584    for (i, mut job) in jobs.into_iter().enumerate() {
6585        job.timeout = job.timeout.saturating_sub(waited_for_lease);
6586        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
6587        if cache.is_some() {
6588            job.prompt.push('\n');
6589            job.prompt
6590                .push_str(&prompt::build_cache_note(node, job.allow_write));
6591        }
6592        let sem = Arc::clone(&sem);
6593        let run = run.to_owned();
6594        let node = node.to_owned();
6595        // Only implementers were told about the task's attachments, so only
6596        // their seats get the directory widened for reading.
6597        let attachments = if node == "implement" {
6598            state.attachments.clone()
6599        } else {
6600            Vec::new()
6601        };
6602        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
6603        // `prompt::build_cache_note`'s doc for why setting it anyway is
6604        // exactly how a sandboxed reviewer's write refusal got reported as a
6605        // defect in the patch, not a property of its own seat. And a
6606        // write-allowed one is handed it only when the lease above was
6607        // actually acquired: a wave that could not get it (`cache_guard` is
6608        // `None`, see its own comment) must not send seats to build into a
6609        // directory this run does not hold - that is the exact concurrent,
6610        // unmanaged-write race this module exists to prevent, not something
6611        // "proceeding anyway" is allowed to reintroduce.
6612        let cache = cache
6613            .filter(|_| job.allow_write && cache_guard.is_some())
6614            .map(Path::to_path_buf);
6615        set.spawn(async move {
6616            let _permit = sem.acquire().await;
6617            let mut seat = job.seat;
6618            let out = agent::invoke(
6619                &job.spec,
6620                &mut seat,
6621                &Invocation {
6622                    cwd: &job.cwd,
6623                    prompt: &job.prompt,
6624                    timeout: job.timeout,
6625                    allow_write: job.allow_write,
6626                    sessions: job.sessions,
6627                    artifacts: &job.artifacts,
6628                    stem: &job.stem,
6629                    run: &run,
6630                    node: &node,
6631                    cache_dir: cache.as_deref(),
6632                    attachments: &attachments,
6633                    writable: &[],
6634                },
6635            )
6636            .await;
6637            let out = match out {
6638                Ok(o) if o.usable() => AgentOutcome::Ok(o),
6639                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
6640                // Billed work the CLI failed to hand over is not an ordinary
6641                // failure, but its text is the CLI's raw error JSON, not an
6642                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
6643                // read it as one by forgetting to check. `usable()` is always
6644                // false here (dropped implies an empty response), so this has
6645                // to be checked before the catch-all `Failed` below or the
6646                // one shape this exists for is lost with the rest.
6647                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
6648                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
6649                Ok(o) => AgentOutcome::Failed(format!(
6650                    "exited with {:?} and no usable output",
6651                    o.exit_code
6652                )),
6653                Err(e) => AgentOutcome::Failed(e.to_string()),
6654            };
6655            (i, seat, out)
6656        });
6657    }
6658    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
6659    while let Some(joined) = set.join_next().await {
6660        let (i, seat, out) = match joined {
6661            Ok(v) => v,
6662            // No seat to clear: a panicked task never reported which one it
6663            // was. The defensive sweep below this loop is what stops that
6664            // seat's `active` entry from surviving forever.
6665            Err(e) => {
6666                tracing::error!("agent task panicked: {e}");
6667                continue;
6668            }
6669        };
6670        state.seat_finished(&seat.key);
6671        record_jobs(state, node, round, &seat.key, &out);
6672        if let Err(e) = state.save() {
6673            tracing::warn!("could not persist a seat's completion: {e:#}");
6674        }
6675        if collected.len() <= i {
6676            collected.resize_with(i + 1, || None);
6677        }
6678        collected[i] = Some((i, seat, out));
6679    }
6680    // Belt-and-braces for the panic branch above: every seat this exact batch
6681    // started shares this `(node, attempt)` pair, and every seat that finished
6682    // normally already cleared itself, so anything left tagged with it here
6683    // can only be a panicked task's leftover. Cleared unconditionally rather
6684    // than left to read as still answering forever.
6685    if state
6686        .active
6687        .values()
6688        .any(|a| a.node == node && a.attempt == attempt)
6689    {
6690        state
6691            .active
6692            .retain(|_, a| !(a.node == node && a.attempt == attempt));
6693        if let Err(e) = state.save() {
6694            tracing::warn!("could not persist the end of a wave: {e:#}");
6695        }
6696    }
6697    // Whether or not the lease above was actually held, several worktrees
6698    // may just have built into the cache with nothing here able to name one
6699    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
6700    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
6701    // clean it might not have strictly needed; trusting a stale match would
6702    // cost it a wrong answer.
6703    if let Some(cache_dir) = cache
6704        && jobs_had_a_writer
6705    {
6706        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
6707    }
6708    if let Some(guard) = cache_guard {
6709        guard.release();
6710    }
6711    collected.into_iter().flatten().collect()
6712}
6713
6714/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
6715/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
6716/// this is data collection, not the fix-specific completion contract in
6717/// [`Runner::continue_fix_report`], and applies regardless of which node
6718/// asked.
6719///
6720/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
6721/// evidence from; `Failed` does not, and correctly contributes nothing — a
6722/// timeout or crash is not itself evidence about a command the seat may have
6723/// started.
6724fn record_jobs(
6725    state: &mut RunState,
6726    node: &str,
6727    round: Option<usize>,
6728    seat: &str,
6729    out: &AgentOutcome,
6730) {
6731    let commands: &[agent::CommandEvidence] = match out {
6732        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
6733        AgentOutcome::Failed(_) => &[],
6734    };
6735    let checked_at = Timestamp::now();
6736    for c in commands {
6737        state.jobs.push(JobRecord {
6738            node: node.to_owned(),
6739            round,
6740            seat: seat.to_owned(),
6741            id: c.id.clone(),
6742            description: c.description.clone(),
6743            checked_at,
6744            status: match c.exit_code {
6745                Some(0) => JobStatus::Completed,
6746                Some(_) => JobStatus::Failed,
6747                None => JobStatus::Unknown,
6748            },
6749            exit_code: c.exit_code,
6750            result_summary: c.result_summary.clone(),
6751            source: c.source.clone(),
6752        });
6753    }
6754}
6755
6756/// Is a review round clean, given how many reviewer seats answered against
6757/// how many the round expected?
6758///
6759/// A seat that never answered (timeout, crash, unparsable output) is not a
6760/// seat that read the patch and found nothing — treating it as such is
6761/// exactly the bug this function exists to close. Under the default `block`
6762/// policy a missing seat can never be clean; `warn` still requires the seats
6763/// that *did* answer to have found nothing blocking and verification to be
6764/// green.
6765///
6766/// `quota_missing` narrows that `block` default for exactly one cause of
6767/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
6768/// a session limit lifts by the very next round buys nothing — the seat is
6769/// asked again with the same quota — so once every missing seat is accounted
6770/// for by a quota loss (and at least one seat *did* answer, so a decision has
6771/// something to rest on) the round is decided on the panel that could answer,
6772/// same as `warn` would. A panel that lost every seat to quota is not
6773/// decided here: `answered == 0` falls through to the existing `block`
6774/// fallback so a fully collapsed panel still waits rather than landing on no
6775/// review at all.
6776fn round_is_clean(
6777    blocking: usize,
6778    e2e_ok: bool,
6779    answered: usize,
6780    expected: usize,
6781    quota_missing: usize,
6782    policy: IncompleteReviewPolicy,
6783) -> bool {
6784    if blocking != 0 || !e2e_ok {
6785        return false;
6786    }
6787    if answered == expected || policy == IncompleteReviewPolicy::Warn {
6788        return true;
6789    }
6790    answered > 0 && expected - answered <= quota_missing
6791}
6792
6793/// The review loop's own conclusion, derived entirely from its persisted
6794/// round records and the round budget that produced them — never from
6795/// `status`, so a reentry (or `gate`/`merge` reading it independently)
6796/// recomputes the identical answer regardless of what an earlier node in the
6797/// same walk, or a previous walk, did to `status`.
6798///
6799/// `None` while more rounds remain to try, including when review never ran
6800/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
6801/// gone clean, or the budget is spent, or the tree has stopped moving (see
6802/// [`STAGNANT_LIMIT`]), the answer is one of two things:
6803///
6804/// - An incomplete panel that raised nothing is missing input, not a
6805///   verified tree — never a hand-off candidate, whatever verification said
6806///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
6807/// - Otherwise, green e2e on the last round hands off (see
6808///   [`Runner::stop_reviewing`]); red e2e blocks.
6809///
6810/// A last round whose own verification is still `ResourceBlocked` — magi
6811/// itself never got a command to run, not evidence the patch is broken —
6812/// is neither: this returns `None` for it too, the same as "more rounds
6813/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
6814/// handling of that shape) instead of this cheap recomputation guessing a
6815/// verdict a real attempt never produced.
6816fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
6817    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
6818        return Some(RunStatus::Gating);
6819    }
6820    let last = reviews.last()?;
6821    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
6822    if reviews.len() < max_rounds && !stagnant {
6823        return None;
6824    }
6825    if last.incomplete() && last.blocking == 0 {
6826        return Some(RunStatus::Blocked);
6827    }
6828    if last.e2e_status() == E2eStatus::ResourceBlocked {
6829        return None;
6830    }
6831    Some(if last.e2e.iter().all(CommandOutcome::ok) {
6832        RunStatus::Gating
6833    } else {
6834        RunStatus::Blocked
6835    })
6836}
6837
6838/// How long a re-ask may take, given the budget the first attempt had.
6839///
6840/// A `nudged` retry is a request to restate an answer the seat has already
6841/// worked out: it carries no new work, so it does not deserve the original
6842/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
6843/// 133 seconds while a third sat for over ten minutes on a resumed session
6844/// holding 410 KB of prior output - and because the retry had inherited the
6845/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
6846/// judging round whose other seats were long finished.
6847///
6848/// A quarter of the budget, with a floor so that a deliberately short timeout
6849/// does not collapse to nothing. A retry that re-sends the whole prompt
6850/// (because the seat kept no context) is the original job again, and keeps the
6851/// original budget.
6852fn retry_budget(full: Duration, nudged: bool) -> Duration {
6853    if nudged {
6854        (full / 4).max(Duration::from_secs(120)).min(full)
6855    } else {
6856        full
6857    }
6858}
6859
6860/// Run a wave and parse each reply, re-asking the seats whose reply was
6861/// unusable.
6862///
6863/// The re-ask is a nudge rather than the whole prompt again when the seat still
6864/// holds its conversation, which is the difference between a cheap retry and
6865/// paying for the entire candidate set twice.
6866///
6867/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
6868/// successor in `roster` is handed to it instead of being re-asked: the
6869/// handover is the retry. A seat with no successor left (a single-agent
6870/// roster, the roster's tail) is nudged as before, up to `retries` times. So
6871/// the asks to one seat in one node number at most
6872/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
6873/// is asked once (a dropped stream is nudged first), and only the last agent
6874/// of the chain gets the `retries` same-agent nudges. Each roster agent is
6875/// tried at most once per seat, walking forward from the seat's own position and never wrapping
6876/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
6877/// other failure stops the chain when the previous agent failed the same way
6878/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
6879/// [`has_context`] is false and the job's own full prompt and full budget are
6880/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
6881/// intermediate ones are not losses) and is returned as a failure like any
6882/// other absent seat — the caller decides whether the panel still has a
6883/// quorum. An empty `roster` disables handover: failures are nudged as they
6884/// always were, and a quota is simply lost. A reply that fails to parse or
6885/// validate is the prompt's doing and is only ever nudged, never handed over.
6886///
6887/// The returned [`SeatState`] names the agent that answered (or tried last).
6888#[allow(clippy::too_many_arguments)]
6889async fn ask_json_wave<T>(
6890    jobs: Vec<SeatJob>,
6891    sem: Arc<Semaphore>,
6892    retries: usize,
6893    roster: &[AgentSpec],
6894    ctx: &WaveCtx<'_>,
6895    losses: &mut Vec<QuotaLoss>,
6896    state: &mut RunState,
6897    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
6898) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6899where
6900    T: serde::de::DeserializeOwned + Send + 'static,
6901{
6902    ask_wave_with(
6903        jobs,
6904        sem,
6905        retries,
6906        roster,
6907        ctx,
6908        losses,
6909        state,
6910        &|text: &str| {
6911            let v = verdict::extract_json::<T>(text)?;
6912            validate(&v)?;
6913            Ok(v)
6914        },
6915    )
6916    .await
6917}
6918
6919/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
6920/// a node whose answer is prose (deliberation) shares the same handover,
6921/// failure classification, quota bookkeeping and bounds instead of a copy.
6922///
6923/// A seat handed to another roster agent is sent the job's `handover` prompt
6924/// (when it has one) rather than `prompt`: the new agent has no session, so a
6925/// resume-style prompt would be incomplete. That holds for the handover ask
6926/// and for every nudge to that agent whose `has_context` is false.
6927#[allow(clippy::too_many_arguments)]
6928async fn ask_wave_with<T>(
6929    jobs: Vec<SeatJob>,
6930    sem: Arc<Semaphore>,
6931    retries: usize,
6932    roster: &[AgentSpec],
6933    ctx: &WaveCtx<'_>,
6934    losses: &mut Vec<QuotaLoss>,
6935    state: &mut RunState,
6936    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
6937) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
6938where
6939    T: Send + 'static,
6940{
6941    let n = jobs.len();
6942    let originals: Vec<SeatJob> = jobs;
6943    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
6944    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
6945    // Nudges each seat's *current* agent has taken — 0 for a first-ask
6946    // answer, N once it has gone through N nudges. Read back once this
6947    // returns, so a caller building a history record (`ReviewRecord`) can
6948    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
6949    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
6950    // field's own doc.
6951    let mut nudges: Vec<usize> = vec![0; n];
6952    // The agent now occupying each seat, the ids it has already been through,
6953    // where in the roster the walk began, the class of the last failure, and
6954    // the stem word of a handover not yet asked (full prompt, full budget).
6955    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
6956    let mut tried: Vec<BTreeSet<String>> = specs
6957        .iter()
6958        .map(|s| BTreeSet::from([s.id.clone()]))
6959        .collect();
6960    let starts: Vec<usize> = specs
6961        .iter()
6962        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
6963        .collect();
6964    let carry = ctx.carry_seats && !roster.is_empty();
6965    // Carried across rounds: ids that failed the seat earlier, and how the
6966    // last failure went (so a repeat of it is not handed over again).
6967    let carried: Vec<BTreeSet<String>> = originals
6968        .iter()
6969        .map(|j| {
6970            state
6971                .seat_history
6972                .get(&j.seat.key)
6973                .filter(|_| carry)
6974                .map(|h| h.failed.clone())
6975                .unwrap_or_default()
6976        })
6977        .collect();
6978    let mut prev: Vec<Option<FailClass>> = originals
6979        .iter()
6980        .map(|j| {
6981            state
6982                .seat_history
6983                .get(&j.seat.key)
6984                .filter(|_| carry)
6985                .and_then(|h| h.last_fail.clone())
6986        })
6987        .collect();
6988    let next_agent = |i: usize, tried: &BTreeSet<String>| -> Option<AgentSpec> {
6989        if carry {
6990            next_for_seat(roster, starts[i], tried, &carried[i]).cloned()
6991        } else {
6992            next_untried_in_roster(roster, starts[i], tried).cloned()
6993        }
6994    };
6995    let mut fresh: Vec<Option<String>> = vec![None; n];
6996    let mut last_quota: Vec<Option<Option<String>>> = vec![None; n];
6997    let mut pending: Vec<usize> = (0..n).collect();
6998
6999    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
7000    // asks: an agent with a successor is asked once and handed over, and only
7001    // a seat with no successor spends `retries` nudges on the same agent. This
7002    // only guarantees the loop's own termination whatever those say.
7003    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
7004    for round in 0..max_rounds {
7005        if pending.is_empty() {
7006            break;
7007        }
7008        let mut batch = Vec::with_capacity(pending.len());
7009        let mut renudged: Vec<&str> = Vec::new();
7010        for &i in &pending {
7011            let src = &originals[i];
7012            // A seat now held by another agent than the job named has no
7013            // session of its own: it gets the full-context prompt whenever
7014            // it is asked in full (the handover ask, a nudge it cannot
7015            // resume).
7016            let full: &str = match &src.handover {
7017                Some(h) if specs[i].id != src.spec.id => h,
7018                _ => &src.prompt,
7019            };
7020            // The prompt and the budget are one decision: a nudge restates
7021            // finished work, a re-sent prompt redoes it.
7022            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
7023                (
7024                    full.to_owned(),
7025                    src.timeout,
7026                    format!("{}-{word}-{}", src.stem, specs[i].id),
7027                )
7028            } else if nudges[i] == 0 {
7029                (full.to_owned(), src.timeout, src.stem.clone())
7030            } else {
7031                renudged.push(src.seat.key.as_str());
7032                let why = done[i]
7033                    .as_ref()
7034                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
7035                    .unwrap_or_else(|| "no parsable answer".to_owned());
7036                let nudge = prompt::nudge(&why);
7037                let nudged = has_context(&specs[i], &seats[i], src.sessions);
7038                let prompt = if nudged {
7039                    nudge
7040                } else {
7041                    format!("{full}\n\n---\n\n{nudge}")
7042                };
7043                (
7044                    prompt,
7045                    retry_budget(src.timeout, nudged),
7046                    format!("{}-retry{}", src.stem, nudges[i]),
7047                )
7048            };
7049            batch.push(SeatJob {
7050                spec: specs[i].clone(),
7051                seat: seats[i].clone(),
7052                cwd: src.cwd.clone(),
7053                prompt,
7054                timeout,
7055                allow_write: src.allow_write,
7056                sessions: src.sessions,
7057                artifacts: src.artifacts.clone(),
7058                stem,
7059                handover: None,
7060            });
7061        }
7062
7063        if !renudged.is_empty() {
7064            state.event(
7065                ctx.node,
7066                format!("retry {round}: re-asking {}", renudged.join(", ")),
7067            );
7068        }
7069        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
7070        let mut still = Vec::new();
7071        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
7072            seats[i] = seat;
7073            let class = FailClass::of(&out);
7074            // A dropped stream is nudged first (the conversation is still
7075            // there to pick up); only a seat whose nudges are spent hands over.
7076            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
7077                && nudges[i] < retries
7078                && !roster.is_empty();
7079            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
7080                let next = should_hand_over(prev[i].as_ref(), &cur)
7081                    .then(|| next_agent(i, &tried[i]))
7082                    .flatten();
7083                if carry {
7084                    let h = state
7085                        .seat_history
7086                        .entry(originals[i].seat.key.clone())
7087                        .or_default();
7088                    h.failed.insert(specs[i].id.clone());
7089                    h.last_fail = Some(cur.clone());
7090                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
7091                        h.last_ok = None;
7092                    }
7093                    // Saved before the next agent is asked, so a restart in
7094                    // between does not forget who failed.
7095                    if let Err(e) = state.save() {
7096                        tracing::warn!("could not persist a seat's failure history: {e:#}");
7097                    }
7098                }
7099                if let Some(next) = next {
7100                    record_handover(
7101                        state,
7102                        ctx.node,
7103                        &originals[i].seat.key,
7104                        &specs[i].id,
7105                        &next.id,
7106                        &cur,
7107                        &fail_reason(&out),
7108                    );
7109                    tried[i].insert(next.id.clone());
7110                    prev[i] = Some(cur.clone());
7111                    seats[i] =
7112                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
7113                    specs[i] = next;
7114                    fresh[i] = Some(cur.stem_word().to_owned());
7115                    nudges[i] = 0;
7116                    done[i] = Some(Err(anyhow::anyhow!(
7117                        "handed over after: {}",
7118                        fail_reason(&out)
7119                    )));
7120                    still.push(i);
7121                    continue;
7122                }
7123            }
7124            if carry
7125                && !nudge_first
7126                && let Some(cur) = class.clone()
7127            {
7128                // The chain ended here (no successor, or a repeated failure).
7129                let h = state
7130                    .seat_history
7131                    .entry(originals[i].seat.key.clone())
7132                    .or_default();
7133                h.failed.insert(specs[i].id.clone());
7134                h.last_fail = Some(cur);
7135            }
7136            let parsed = match out {
7137                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
7138                AgentOutcome::Quota(o) => {
7139                    last_quota[i] = Some(o.quota.as_ref().and_then(|q| q.reset.clone()));
7140                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
7141                }
7142                // Not a parseable answer: the nudge loop re-asks it, which is
7143                // exactly what a dropped stream needs. Just don't hand its raw
7144                // error JSON to `extract_json`.
7145                AgentOutcome::Dropped(o) => {
7146                    let why = o
7147                        .dropped
7148                        .as_ref()
7149                        .map(|d| d.why.as_str())
7150                        .unwrap_or("the CLI ended the stream without delivering its answer");
7151                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
7152                }
7153                AgentOutcome::Failed(e) => Err(anyhow::anyhow!(e)),
7154            };
7155            let quota = class == Some(FailClass::Quota);
7156            let failed = parsed.is_err();
7157            done[i] = Some(parsed);
7158            if carry && !failed {
7159                let h = state
7160                    .seat_history
7161                    .entry(originals[i].seat.key.clone())
7162                    .or_default();
7163                h.failed.remove(&specs[i].id);
7164                h.last_ok = Some(specs[i].id.clone());
7165                h.last_fail = None;
7166            }
7167            // Do not re-ask a rate-limited seat (quota) — a retry is known to
7168            // fail the same way; and never re-ask a seat that already parsed.
7169            // A failed agent that still has a successor is not re-asked
7170            // either: the handover was its remedy and has just been refused
7171            // (the chain stops on a repeated failure class). A seat with no
7172            // successor left (a single-agent roster, the roster's tail, or an
7173            // empty roster) keeps the same-agent nudge, bounded by `retries`.
7174            let agent_failure = class.is_some()
7175                && !nudge_first
7176                && !roster.is_empty()
7177                && next_agent(i, &tried[i]).is_some();
7178            if failed && !quota && !agent_failure && nudges[i] < retries {
7179                nudges[i] += 1;
7180                still.push(i);
7181            }
7182        }
7183        pending = still;
7184    }
7185
7186    // One loss per seat whose chain ended on a quota: the intermediate ones
7187    // were absorbed by a handover and are not losses.
7188    for (i, q) in last_quota.into_iter().enumerate() {
7189        if let Some(reset) = q {
7190            losses.push(QuotaLoss {
7191                seat: originals[i].seat.key.clone(),
7192                node: ctx.node.to_owned(),
7193                at: Timestamp::now(),
7194                reset,
7195            });
7196        }
7197    }
7198
7199    seats
7200        .into_iter()
7201        .zip(done)
7202        .zip(nudges)
7203        .map(|((seat, res), attempts)| {
7204            (
7205                seat,
7206                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
7207                attempts,
7208            )
7209        })
7210        .collect()
7211}
7212
7213/// Acquire the shared build cache's lease, waiting out contention within
7214/// `budget` (never past it — see AGENTS.md's build-cache section on why an
7215/// unbounded wait is never acceptable).
7216///
7217/// A first, non-blocking check happens before ever waiting; if it finds the
7218/// lease busy, that fact is logged as a `verify` event *and* flushed with
7219/// [`RunState::save`] immediately — not only once the wait finally succeeds
7220/// or gives up — so a `magi show` run by a different process while this one
7221/// is still waiting reads a `run.json` that says so, rather than whatever it
7222/// looked like before the wait started. The same applies to the terminal
7223/// failure: logged and saved before this returns `Err`, so a caller that
7224/// could not get the lease at all still leaves a legible record of why.
7225async fn acquire_cache_lease(
7226    state: &mut RunState,
7227    cache_dir: &Path,
7228    owner: &crate::cache::Owner,
7229    budget: Duration,
7230    context: &str,
7231) -> Result<crate::cache::Guard> {
7232    let home = crate::run::home();
7233    let started = Instant::now();
7234    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
7235        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
7236        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
7237        Err(e) => {
7238            state.event(
7239                "verify",
7240                format!("{context}: could not check the shared build cache: {e:#}"),
7241            );
7242            if let Err(e2) = state.save() {
7243                tracing::warn!("could not persist a cache-check failure: {e2:#}");
7244            }
7245            return Err(e);
7246        }
7247    };
7248    state.event(
7249        "verify",
7250        format!(
7251            "{context}: waiting for the shared build cache at {} ({})",
7252            cache_dir.display(),
7253            busy.describe()
7254        ),
7255    );
7256    if let Err(e) = state.save() {
7257        tracing::warn!("could not persist a cache wait: {e:#}");
7258    }
7259    let remaining = budget.saturating_sub(started.elapsed());
7260    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
7261        Ok(g) => Ok(g),
7262        Err(e) => {
7263            state.event("verify", format!("{context}: {e:#}"));
7264            if let Err(e2) = state.save() {
7265                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
7266            }
7267            Err(e)
7268        }
7269    }
7270}
7271
7272/// Run `body` — a verify command batch — while holding the shared build
7273/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
7274/// never interleave with another borrower's build against the same
7275/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
7276/// timeout, or a human's own `magi review`. See the `cache` module doc for
7277/// why this matters more than Cargo's own per-target locking covers — two
7278/// *different* worktrees building the same package name/version into one
7279/// cache directory is a staleness bug, not a lock contention one.
7280///
7281/// The wait for the lease is carved out of `budget`, never on top of it —
7282/// `body` is handed whatever is left, so a caller's own node timeout is the
7283/// only clock involved, exactly what AGENTS.md's build-cache section asks
7284/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
7285/// cache configured at all — this is a pass-through: `body` runs with the
7286/// full budget and nothing is leased.
7287///
7288/// A lease that cannot be acquired within `budget` is reported as a single
7289/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
7290/// verification — the same shape a spawn failure already takes in
7291/// [`run_commands`], so a caller need not special-case it.
7292#[allow(clippy::too_many_arguments)]
7293async fn with_cache_lease<'s, F, Fut>(
7294    state: &'s mut RunState,
7295    cache_dir: Option<&Path>,
7296    node: &str,
7297    seat: &str,
7298    worktree: &Path,
7299    head: &str,
7300    budget: Duration,
7301    context: &str,
7302    body: F,
7303) -> (Vec<CommandOutcome>, bool)
7304where
7305    F: FnOnce(&'s mut RunState, Duration) -> Fut,
7306    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
7307{
7308    let Some(cache_dir) = cache_dir else {
7309        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
7310        return (outcomes, retried);
7311    };
7312    let home = crate::run::home();
7313    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
7314    let started = Instant::now();
7315    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
7316        Ok(g) => g,
7317        Err(e) => {
7318            return (
7319                vec![CommandOutcome {
7320                    command: "(waiting for the shared build cache)".to_owned(),
7321                    code: None,
7322                    output_tail: e.to_string(),
7323                    duration_ms: started.elapsed().as_millis() as u64,
7324                    resource_blocked: true,
7325                }],
7326                false,
7327            );
7328        }
7329    };
7330    let identity = crate::cache::Identity::new(worktree, head);
7331    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
7332        // A failed freshness check means this process cannot vouch for what
7333        // is sitting in the cache right now - on Windows this is exactly the
7334        // "a stale test executable is still locked, `cargo clean -p` cannot
7335        // remove it" case the evidence log records. Running verify anyway
7336        // and reporting whatever it says would let a result nobody can trust
7337        // stand for the tree it claims to have checked; fail the step
7338        // instead of the patch.
7339        state.event(
7340            "verify",
7341            format!(
7342                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
7343                worktree.display(),
7344                short(head)
7345            ),
7346        );
7347        guard.release();
7348        return (
7349            vec![CommandOutcome {
7350                command: "(confirming the shared build cache is fresh)".to_owned(),
7351                code: None,
7352                output_tail: e.to_string(),
7353                duration_ms: started.elapsed().as_millis() as u64,
7354                resource_blocked: true,
7355            }],
7356            false,
7357        );
7358    }
7359    let remaining = budget.saturating_sub(started.elapsed());
7360    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
7361    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
7362    // `start_kill`); confirm it actually has before handing the directory to
7363    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
7364    // for what this can and cannot see.
7365    if !timed_out_pids.is_empty() {
7366        wait_for_timed_out_children_to_die(&timed_out_pids).await;
7367    }
7368    guard.release();
7369    (outcomes, retried)
7370}
7371
7372/// Poll `pids` — commands [`run_commands`] reports as still running when its
7373/// own timeout elapsed — until every one is confirmed gone, or
7374/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
7375///
7376/// Real confirmation where confirmation is possible, not a substitute for
7377/// full process-tree observation: a grandchild the timed-out process spawned
7378/// and that survives independently of it is invisible to a pid check the
7379/// same way it always was, and continuing to observe and collect *that*
7380/// stays a different piece of work with its own owner. This only narrows a
7381/// fixed blind wait into an actual check of the pids this process does know
7382/// about.
7383async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
7384    wait_for_pids_with(
7385        pids,
7386        crate::proc::pid_alive,
7387        LEASE_RELEASE_POLL,
7388        LEASE_RELEASE_MAX_WAIT,
7389    )
7390    .await;
7391}
7392
7393/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
7394/// interval and ceiling supplied by the caller, so the polling *logic* -
7395/// returns as soon as every pid reports dead, gives up at the ceiling
7396/// otherwise - is testable on millisecond durations without asking the real
7397/// OS about a pid at all.
7398async fn wait_for_pids_with<F: Fn(u32) -> bool>(
7399    pids: &[u32],
7400    alive: F,
7401    poll: Duration,
7402    max_wait: Duration,
7403) {
7404    let deadline = Instant::now() + max_wait;
7405    loop {
7406        if pids.iter().all(|&pid| !alive(pid)) {
7407            return;
7408        }
7409        if Instant::now() >= deadline {
7410            return;
7411        }
7412        tokio::time::sleep(poll).await;
7413    }
7414}
7415
7416/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
7417/// admission that it could not even get a verify command to run, as opposed
7418/// to evidence the command actually produced? A caller that would otherwise
7419/// read a resource-blocked outcome as a red command must check this first:
7420/// see [`Runner::gate`], which retries rather than records `Blocked` when
7421/// this is true.
7422fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
7423    outcomes.iter().any(|o| o.resource_blocked)
7424}
7425
7426/// What [`Runner::gate_fix_round`] decided.
7427enum GateFix {
7428    /// The tree changed and `verify.e2e` is still green: run the gate again.
7429    Retry,
7430    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
7431    /// last failure stands and the run ends blocked.
7432    Stop,
7433    /// `verify.e2e` could not run after the fix (magi's own contention):
7434    /// decide nothing now, a later reentry retries.
7435    Defer,
7436}
7437
7438/// Is every red command in `outcomes` an ordinary failure the code could
7439/// explain: it ran, exited non-zero, and said something?
7440///
7441/// A timeout, a spawn failure and a killed process all leave `code` `None`;
7442/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
7443/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
7444/// what the command is stays the gate's business.
7445fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
7446    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
7447    red.peek().is_some()
7448        && red.all(|o| {
7449            !o.resource_blocked
7450                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
7451                && !o.output_tail.trim().is_empty()
7452        })
7453}
7454
7455/// Describe one verify command's outcome for the event log, distinguishing a
7456/// build/link failure — the toolchain never produced a binary to run — from
7457/// an actual test failure, since only the latter is a verdict on the patch.
7458fn e2e_outcome_label(o: &CommandOutcome) -> String {
7459    if o.ok() {
7460        return "pass".to_owned();
7461    }
7462    let reason = if o.build_failed() {
7463        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
7464    } else {
7465        format!("FAIL ({:?})", o.code)
7466    };
7467    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
7468}
7469
7470/// Run `verify.e2e`, retrying once if the first attempt could not build or
7471/// link — a build/link failure is frequently a race against a shared
7472/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
7473/// `verify` event per command, tagged with `context` (normally `"round N"`)
7474/// so the two call sites that need this — the ordinary per-round leg in
7475/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
7476/// it will ever call a round green — read identically in the event log.
7477async fn run_e2e_with_retry(
7478    state: &mut RunState,
7479    shell: &[String],
7480    commands: &[String],
7481    worktree: &Path,
7482    timeout: Duration,
7483    context: &str,
7484) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
7485    let (mut e2e, mut timed_out_pids) = run_commands(
7486        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
7487    )
7488    .await;
7489    for o in &e2e {
7490        state.event(
7491            "verify",
7492            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
7493        );
7494    }
7495    // A build/link failure is not a verdict on the patch — it is frequently a
7496    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
7497    // one retry before letting a red like that decide the round.
7498    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
7499    if verify_retried {
7500        state.event(
7501            "verify",
7502            format!(
7503                "{context}: verify could not build/link, not a test result — retrying once \
7504                 before concluding"
7505            ),
7506        );
7507        let retried = run_commands(
7508            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
7509        )
7510        .await;
7511        e2e = retried.0;
7512        // Both attempts' timeouts matter, not just the last one: the first
7513        // attempt's descendants may still be alive alongside the retry's.
7514        timed_out_pids.extend(retried.1);
7515        for o in &e2e {
7516            state.event(
7517                "verify",
7518                format!(
7519                    "{context}: retry `{}` -> {}",
7520                    o.command,
7521                    e2e_outcome_label(o)
7522                ),
7523            );
7524        }
7525    }
7526    (e2e, verify_retried, timed_out_pids)
7527}
7528
7529/// Run configured shell commands in `cwd`, in order. The second element is
7530/// the pid of every command that hit `timeout` and was still running when
7531/// this stopped waiting on it (best-effort: `None` when the platform did not
7532/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
7533/// that releases a shared resource afterward needs to know.
7534///
7535/// Records `task` into [`RunState::active`] at every command boundary
7536/// (`RunState::task_command`) and clears it once the whole list has run
7537/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
7538/// for minutes with no seat and no output of its own to show for it (see
7539/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
7540/// apart from "not yet run" without this), and this is the only place that
7541/// knows which command is running right now and how many are left. Three
7542/// saves per command — start, not per second — matching the same "only at a
7543/// boundary" rule [`wave`] already follows for seats.
7544#[allow(clippy::too_many_arguments)]
7545async fn run_commands(
7546    state: &mut RunState,
7547    node: &str,
7548    task: &str,
7549    attempt: usize,
7550    shell: &[String],
7551    commands: &[String],
7552    cwd: &Path,
7553    timeout: Duration,
7554) -> (Vec<CommandOutcome>, Vec<u32>) {
7555    if commands.is_empty() {
7556        // Nothing to mark as running and nothing to clear — an empty list
7557        // means "not configured", and touching `active` (or the disk) over
7558        // that would be a write for every round of a repo with no
7559        // `verify.e2e` / `verify.gate` commands at all.
7560        return (Vec::new(), Vec::new());
7561    }
7562    let mut out = Vec::new();
7563    let mut timed_out_pids = Vec::new();
7564    let total = commands.len();
7565    for (idx, command) in commands.iter().enumerate() {
7566        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
7567        if let Err(e) = state.save() {
7568            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
7569        }
7570        let started = Instant::now();
7571        let mut cmd = tokio::process::Command::new(&shell[0]);
7572        cmd.quiet();
7573        cmd.args(&shell[1..])
7574            .arg(command)
7575            .current_dir(cwd)
7576            .stdin(std::process::Stdio::null())
7577            .stdout(std::process::Stdio::piped())
7578            .stderr(std::process::Stdio::piped())
7579            .kill_on_drop(true);
7580        let spawned = cmd.spawn();
7581        let (code, body) = match spawned {
7582            Ok(child) => {
7583                // Captured before the child is consumed below: `kill_on_drop`
7584                // only *asks* the process to die when the timeout branch
7585                // drops it, and the pid is the only way anyone downstream can
7586                // later check whether that request actually took.
7587                let pid = child.id();
7588                match tokio::time::timeout(timeout, child.wait_with_output()).await {
7589                    Ok(Ok(o)) => {
7590                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
7591                        body.push_str(&String::from_utf8_lossy(&o.stderr));
7592                        (o.status.code(), body)
7593                    }
7594                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
7595                    Err(_) => {
7596                        if let Some(pid) = pid {
7597                            timed_out_pids.push(pid);
7598                        }
7599                        (None, format!("timed out after {}s", timeout.as_secs()))
7600                    }
7601                }
7602            }
7603            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
7604        };
7605        out.push(CommandOutcome {
7606            command: command.clone(),
7607            code,
7608            output_tail: tail(&body, OUTPUT_TAIL),
7609            duration_ms: started.elapsed().as_millis() as u64,
7610            resource_blocked: false,
7611        });
7612    }
7613    state.task_finished(task);
7614    if let Err(e) = state.save() {
7615        tracing::warn!("could not persist the end of {task}: {e:#}");
7616    }
7617    (out, timed_out_pids)
7618}
7619
7620/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
7621/// section (`report::run`) and in the `merge` event this node records — for
7622/// the operator to run by hand.
7623///
7624/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
7625/// branch whose ruleset forbids merge commits (GitHub's "must not contain
7626/// merge commits", or "require linear history") rejects the push a `--no-ff`
7627/// merge would produce, which is exactly the guidance this function replaces.
7628/// `message`'s first line becomes the squash commit's subject, matching the
7629/// note `report::run` prints alongside this command — see that function for
7630/// why an explicit subject is not optional there.
7631fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
7632    let repo = repo.display();
7633    match style {
7634        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
7635        MergeStyle::Squash => {
7636            // The subject sits inside double quotes, and a title an agent
7637            // wrote may carry the characters that break out of them.
7638            let subject = message
7639                .lines()
7640                .next()
7641                .unwrap_or(branch)
7642                .replace(['\\', '"', '$', '`'], "");
7643            format!(
7644                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
7645            )
7646        }
7647        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
7648    }
7649}
7650
7651/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
7652/// under the hood, rejects a `title` over 256 characters and the whole
7653/// command fails — no PR at all, for a run whose body was otherwise fine
7654/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
7655/// that limit: `title_from` counts `chars()` (Unicode scalars), which is not
7656/// always how GitHub counts, plus one character for the trailing ellipsis
7657/// `title_from` may add. It is a margin, not a guarantee — a title packed
7658/// with multi-unit characters could still in principle land close to the
7659/// edge, but a real task title's occasional emoji or accented letter fits
7660/// comfortably inside it.
7661const PR_TITLE_MAX: usize = 240;
7662
7663/// What `merge = "pr"` (and the merge commit of the other modes) says about a
7664/// change: a title and a body describing what was *implemented*, not the task
7665/// that asked for it. A task reads as a request; a reader of the merged
7666/// history wants the change.
7667struct PrMessage {
7668    title: String,
7669    body: String,
7670}
7671
7672impl PrMessage {
7673    /// Title, blank line, body. The first line is the squash/merge commit
7674    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
7675    /// has to stay one sensible line.
7676    fn commit_message(&self) -> String {
7677        format!("{}\n\n{}", self.title, self.body)
7678    }
7679}
7680
7681/// The text after a leading `TITLE:` (any case) on `line`.
7682fn title_marker(line: &str) -> Option<&str> {
7683    let line = line.trim();
7684    let head = line.get(..6)?;
7685    head.eq_ignore_ascii_case("title:")
7686        .then(|| line[6..].trim())
7687}
7688
7689/// The implementer's own one-line title: the `TITLE:` line the implement
7690/// prompt asks for at the top of its SUMMARY. Candidate commits are all
7691/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
7692/// source, and a title that says as much is refused here too.
7693fn summary_title(summary: &str) -> Option<String> {
7694    let first = summary.lines().find(|l| !l.trim().is_empty())?;
7695    let raw = title_marker(first)?;
7696    if raw.is_empty() {
7697        return None;
7698    }
7699    let title = queue::title_from(raw, PR_TITLE_MAX);
7700    let lower = title.to_ascii_lowercase();
7701    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7702        return None;
7703    }
7704    Some(title)
7705}
7706
7707/// How `open_review`'s instruction begins; see [`landing_title`].
7708const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
7709
7710/// Marker `open_review` gives a candidate that nothing in the roster wrote.
7711const EXISTING_BRANCH: &str = "(existing branch)";
7712
7713/// Does this run review work that already existed, rather than implement a
7714/// task? Runs recorded before `reviewed_commits` existed carry only the
7715/// candidate marker.
7716fn is_review_run(state: &RunState) -> bool {
7717    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
7718}
7719
7720/// The title of a review-only run: the subject of the oldest commit under
7721/// review. Later commits are usually fixups, and `instruction` is the review
7722/// prompt, which says nothing about the change. GitHub text is English, so a
7723/// non-ASCII or blank subject yields `None` and the caller's neutral title.
7724fn review_title(state: &RunState) -> Option<String> {
7725    english_subject(state.reviewed_commits.as_ref()?.first()?)
7726}
7727
7728/// `raw` as a pull request title, or `None` when it is blank, not English
7729/// (GitHub text is), or one of magi's own candidate commit subjects.
7730fn english_subject(raw: &str) -> Option<String> {
7731    let raw = raw.trim();
7732    if raw.is_empty() || !raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic()) {
7733        return None;
7734    }
7735    let title = queue::title_from(raw, PR_TITLE_MAX);
7736    let lower = title.to_ascii_lowercase();
7737    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
7738        return None;
7739    }
7740    Some(title)
7741}
7742
7743/// What a review-only run's branch says about itself, read at the moment the
7744/// pull request is opened.
7745#[derive(Debug, Clone, PartialEq, Eq)]
7746struct BranchFacts {
7747    /// `(subject, body)` of each commit, oldest first.
7748    commits: Vec<(String, String)>,
7749    /// Trimmed `git diff --stat`.
7750    stat: String,
7751}
7752
7753/// Longest diff stat shown: this many file lines plus the summary line.
7754const STAT_FILE_LINES: usize = 25;
7755/// Cap on the commit list, well inside GitHub's 65536-character body limit.
7756const COMMITS_MAX_CHARS: usize = 20_000;
7757
7758/// Read the commits and diff stat of `base..branch`. `None` when git cannot
7759/// say or finds nothing, so the caller falls back to what the run recorded.
7760async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
7761    let commits = git::commit_log(repo, base, branch).await.ok()?;
7762    if commits.is_empty() {
7763        return None;
7764    }
7765    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
7766    let lines: Vec<&str> = stat.lines().collect();
7767    let stat = if lines.len() > STAT_FILE_LINES + 1 {
7768        let omitted = lines.len() - 1 - STAT_FILE_LINES;
7769        let more = format!(" ... {omitted} more file(s)");
7770        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
7771        kept.push(&more);
7772        kept.push(lines[lines.len() - 1]);
7773        kept.join("\n")
7774    } else {
7775        lines.join("\n")
7776    };
7777    Some(BranchFacts { commits, stat })
7778}
7779
7780/// Defang what would break the surrounding markdown: a closing `</details>`
7781/// and a code fence.
7782fn markdown_safe(text: &str) -> String {
7783    text.replace("</details>", "&lt;/details&gt;")
7784        .replace("\x60\x60\x60", "~~~")
7785}
7786
7787fn neutral_title(state: &RunState, winner: char) -> String {
7788    format!(
7789        "chore: land candidate {} of run {}",
7790        winner.to_ascii_uppercase(),
7791        state.id
7792    )
7793}
7794
7795/// The pull request title to hand to `land::merge_subject`. A review-only run
7796/// opened by an earlier build titled its pull request with the review prompt;
7797/// that title is dropped (empty, so the fallback applies) rather than landed.
7798/// Any other title, including an operator's rename, passes through untouched,
7799/// and so does every title of a run that implements a task.
7800pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
7801    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
7802        ""
7803    } else {
7804        pr_title
7805    }
7806}
7807
7808/// What the squash subject falls back to when the pull request title is empty
7809/// or candidate-shaped: for a review-only run the derived title, never the
7810/// review prompt held in `instruction`.
7811pub fn landing_subject_source(state: &RunState) -> String {
7812    if is_review_run(state) {
7813        let winner = state.candidates.first().map_or('A', |c| c.label);
7814        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
7815    }
7816    state.instruction.clone()
7817}
7818
7819/// `summary` without its `TITLE:` line, which the pull request title already
7820/// carries.
7821fn summary_without_title(summary: &str) -> String {
7822    let mut lines = summary.trim().lines().peekable();
7823    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
7824        lines.next();
7825    }
7826    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
7827}
7828
7829/// The pull request title and body for the winning candidate.
7830///
7831/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
7832/// the task's own opening line via [`queue::title_from`] when there is none.
7833/// `state.instruction` can open with blank lines (`task_text` only rejects a
7834/// body that is blank *entirely*), which `title_from` skips.
7835///
7836/// Body: the implementer's summary and the fixer's notes, then — when the
7837/// winning review round was not clean — the findings still open and whatever
7838/// the fixer declined, so `merge = "pr"` hands the reader the same material
7839/// `magi show` does. The task follows inside a collapsed block, and the
7840/// footer repeats the run and candidate as plain tags for a reader holding
7841/// only the merged commit or the PR body.
7842#[cfg(test)]
7843fn pr_message(state: &RunState, winner: char) -> PrMessage {
7844    pr_message_with(state, winner, None)
7845}
7846
7847/// [`pr_message`] with what the branch of a review-only run says about itself.
7848/// `facts` is ignored for a run that implements a task.
7849fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
7850    let summary = state
7851        .candidates
7852        .iter()
7853        .find(|c| c.label == winner)
7854        .map(|c| c.summary.as_str())
7855        .unwrap_or_default();
7856    // The fallback is the operator's own words and may not be English; GitHub
7857    // text always is, so a non-English task gets a neutral title instead.
7858    let review = is_review_run(state);
7859    let title = if review {
7860        facts
7861            .and_then(|f| english_subject(&f.commits.first()?.0))
7862            .or_else(|| review_title(state))
7863            .or_else(|| {
7864                state
7865                    .candidates
7866                    .iter()
7867                    .find(|c| c.label == winner)
7868                    .filter(|c| !c.branch.starts_with("magi/"))
7869                    .and_then(|c| english_subject(&c.branch))
7870            })
7871            .unwrap_or_else(|| neutral_title(state, winner))
7872    } else {
7873        summary_title(summary).unwrap_or_else(|| {
7874            let t = queue::title_from(&state.instruction, PR_TITLE_MAX);
7875            if t.is_ascii() && t.chars().any(|c| c.is_ascii_alphabetic()) {
7876                t
7877            } else {
7878                neutral_title(state, winner)
7879            }
7880        })
7881    };
7882
7883    let mut body = String::new();
7884    let what = summary_without_title(summary);
7885    if !what.is_empty() {
7886        body.push_str("## Summary\n\n");
7887        body.push_str(&what);
7888        body.push_str("\n\n");
7889    }
7890
7891    // The last round is usually a clean verification pass with no fix of its
7892    // own, so every round's notes are read, not just the final one's.
7893    let notes: Vec<(usize, &str)> = state
7894        .reviews
7895        .iter()
7896        .filter_map(|r| {
7897            let n = r.fix.as_ref()?.notes.trim();
7898            (!n.is_empty()).then_some((r.round, n))
7899        })
7900        .collect();
7901    if !notes.is_empty() {
7902        body.push_str("## Review fixes\n\n");
7903        if let [(_, only)] = notes.as_slice() {
7904            body.push_str(only);
7905            body.push_str("\n\n");
7906        } else {
7907            for (round, n) in &notes {
7908                body.push_str(&format!("### Round {round}\n\n{n}\n\n"));
7909            }
7910        }
7911    }
7912    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
7913
7914    let open = state.open_findings();
7915    if !open.is_empty() {
7916        body.push_str("## Open review findings\n\n");
7917        for f in &open {
7918            body.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
7919        }
7920        body.push('\n');
7921    }
7922
7923    if let Some(fix) = fix
7924        && !fix.rejected.is_empty()
7925    {
7926        body.push_str("## Declined by the fixer\n\n");
7927        for r in &fix.rejected {
7928            body.push_str(&format!("- `{}`: {}\n", r.id, r.why));
7929        }
7930        body.push('\n');
7931    }
7932
7933    if review {
7934        // The review prompt is not the task; list what the branch carries.
7935        body.push_str("## Commits under review\n\n");
7936        if let Some(facts) = facts {
7937            let mut left = COMMITS_MAX_CHARS;
7938            for (i, (subject, text)) in facts.commits.iter().enumerate() {
7939                let mut entry = format!("- {}\n", markdown_safe(subject));
7940                for l in markdown_safe(text).lines() {
7941                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
7942                }
7943                if left == 0 {
7944                    body.push_str(&format!(
7945                        "- ... {} more commit(s)\n",
7946                        facts.commits.len() - i
7947                    ));
7948                    break;
7949                }
7950                if entry.len() > left {
7951                    // Even the first commit is cut: one huge body must not
7952                    // push the whole description past GitHub's limit.
7953                    let mut end = left;
7954                    while !entry.is_char_boundary(end) {
7955                        end -= 1;
7956                    }
7957                    entry.truncate(end);
7958                    entry.push_str("\n  ... (truncated)\n");
7959                    left = 0;
7960                } else {
7961                    left -= entry.len();
7962                }
7963                body.push_str(&entry);
7964            }
7965            if !facts.stat.trim().is_empty() {
7966                body.push_str(&format!(
7967                    "\n## Diff stat\n\n```\n{}\n```\n",
7968                    markdown_safe(facts.stat.trim())
7969                ));
7970            }
7971        } else {
7972            match &state.reviewed_commits {
7973                Some(subjects) => {
7974                    for s in subjects {
7975                        body.push_str(&format!("- {}\n", s.trim()));
7976                    }
7977                }
7978                None => {
7979                    // An older run kept only the prompt, with the commit list
7980                    // after its first paragraph.
7981                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
7982                    body.push_str(rest.trim());
7983                    body.push('\n');
7984                }
7985            }
7986        }
7987    } else {
7988        let task = state.instruction.trim();
7989        let task = if task.is_empty() {
7990            "(empty task)"
7991        } else {
7992            task
7993        };
7994        body.push_str(&format!(
7995            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
7996            task.replace("</details>", "&lt;/details&gt;")
7997        ));
7998    }
7999
8000    body.push_str(&format!(
8001        "\n---\nmagi:run/{} magi:candidate-{}\n",
8002        state.id,
8003        winner.to_ascii_lowercase()
8004    ));
8005
8006    // Prompts are advisory; this is the enforced half of the confidentiality
8007    // rule, and it covers the verbatim task in <details> too.
8008    let id = crate::scrub::Identity::current();
8009    PrMessage {
8010        title: crate::scrub::scrub(&title, &id),
8011        body: crate::scrub::scrub(&body, &id),
8012    }
8013}
8014
8015/// The task with what the repository says about the existing work it names
8016/// appended, so an implementer knows what it started from and what it must
8017/// not redo. Unchanged when the task names nothing.
8018fn seeded_instruction(state: &RunState) -> String {
8019    match refs::describe(&state.seeds) {
8020        Some(facts) => format!(
8021            "{}\n\n# Existing work the task refers to\n\n{facts}\n\n\
8022             Candidates start from the unmerged branch named above, when there \
8023             is one, and carry any unmerged commit named by sha as a \
8024             cherry-pick. Check that this is what the task meant before \
8025             building on it.",
8026            state.instruction
8027        ),
8028        None => state.instruction.clone(),
8029    }
8030}
8031
8032/// Does the winner have no commits ahead of the base it would land on?
8033/// Any failure to find out reads as "not empty": the merge then behaves as it
8034/// always did rather than refusing on a guess.
8035async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
8036    let base = &state.base_branch;
8037    let mut against = base.clone();
8038    if mode == MergeMode::Pr {
8039        let remote = &state.config.merge.remote;
8040        let tracking = format!("{remote}/{base}");
8041        let fetched = git::fetch(repo, remote, base).await;
8042        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
8043            against = tracking;
8044        }
8045    }
8046    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
8047}
8048
8049/// Why nothing was opened for an empty winner, with what the task's own
8050/// references resolved to.
8051fn empty_candidate_detail(state: &RunState, base: &str) -> String {
8052    let mut detail = format!(
8053        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
8054         nothing to open a pull request for"
8055    );
8056    match refs::describe(&state.seeds) {
8057        Some(facts) => detail.push_str(&format!("\nReferences in the task:\n{facts}")),
8058        None => detail.push_str(
8059            "\nThe task names no existing branch or commit; if it means to land work \
8060             that lives elsewhere, name the branch (magi/<run>/<label>) or the sha.",
8061        ),
8062    }
8063    detail
8064}
8065
8066/// What the `Pr` merge does once it knows whether the branch already has an
8067/// open pull request.
8068#[derive(Debug, PartialEq, Eq)]
8069enum PrPlan {
8070    Create,
8071    Adopt { url: String, title: String },
8072    Stop(String),
8073}
8074
8075/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
8076/// a failed lookup -> stop with the real reason. Never guesses.
8077fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
8078    match found {
8079        Ok(land::OpenPr::None) => PrPlan::Create,
8080        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
8081        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
8082            "several open pull requests exist for this branch, not picking one: {}",
8083            urls.join(" ")
8084        )),
8085        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
8086    }
8087}
8088
8089/// `gh pr create`, returning the PR url.
8090async fn gh_pr_create(
8091    cwd: &Path,
8092    base: &str,
8093    head: &str,
8094    title: &str,
8095    body: &str,
8096) -> Result<String> {
8097    let out = tokio::process::Command::new("gh")
8098        .args([
8099            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
8100        ])
8101        .current_dir(cwd)
8102        .quiet()
8103        .stdin(std::process::Stdio::null())
8104        .output()
8105        .await
8106        .context("spawn gh")?;
8107    if out.status.success() {
8108        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
8109    } else {
8110        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
8111    }
8112}
8113
8114/// Tear a run's worktrees and branches down.
8115///
8116/// `home` is where the updated `run.json` is saved (via
8117/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
8118/// a housekeeping pass already has its own honest `home` handed to it, and
8119/// falling through to the global here would write back through whichever
8120/// directory some other process or test pinned into that `OnceLock` first,
8121/// not the one the caller actually resolved its `runs` and `state` from.
8122pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
8123    let repo = state.repo.clone();
8124    let root = state.worktree_root();
8125    let winner = state.tally.as_ref().map(|t| t.winner);
8126    let mut removed = Vec::new();
8127
8128    for i in 0..state.candidates.len() {
8129        let c = state.candidates[i].clone();
8130        let is_winner = Some(c.label) == winner;
8131        if is_winner && !drop_winner {
8132            continue;
8133        }
8134        if c.worktree.exists() {
8135            git::worktree_remove(&repo, &c.worktree).await.ok();
8136            removed.push(c.worktree.to_string_lossy().into_owned());
8137        }
8138        // A branch handed to a later run (and its pull request) is not this
8139        // run's to delete.
8140        let handed_over = state.released_branches.contains(&c.branch);
8141        if !handed_over && git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
8142            git::branch_delete(&repo, &c.branch).await.ok();
8143            removed.push(c.branch.clone());
8144        }
8145        state.candidates[i].folded = true;
8146    }
8147
8148    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
8149        let path = name.path();
8150        let keep = !drop_winner
8151            && winner.is_some_and(|w| {
8152                path.file_name()
8153                    .is_some_and(|n| n == format!("cand-{w}").as_str())
8154            });
8155        if keep {
8156            continue;
8157        }
8158        git::worktree_remove(&repo, &path).await.ok();
8159        removed.push(path.to_string_lossy().into_owned());
8160    }
8161
8162    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
8163    // judge worktrees, so once the loop above has cleared all of them out,
8164    // the parent is a bare directory nobody else was ever going to remove -
8165    // git only ever managed what was inside it. Left alone, one of these
8166    // accumulates per fully-folded run; the operator's own machine had 74.
8167    // `remove_if_empty` re-checks rather than assuming: a run whose winner
8168    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
8169    // so does anything a run never claimed that happens to share the bay.
8170    remove_if_empty(&root);
8171
8172    if state.enabled_worktree_config && drop_winner {
8173        // A release, not a raw disable: some sibling run in this repository
8174        // may still hold its own reference (see `git::acquire_worktree_config`),
8175        // and only the last release actually turns the setting back off.
8176        git::release_worktree_config(&repo).await.ok();
8177        state.enabled_worktree_config = false;
8178    }
8179    state.save_under(home)?;
8180    Ok(removed)
8181}
8182
8183/// Remove `dir` if it exists and has nothing in it.
8184///
8185/// Best-effort and silent by design: a directory that is not empty (a run
8186/// whose winner is still parked there, a stray file some other process left)
8187/// is exactly the case this must refuse, and a directory that is already gone
8188/// is not a failure worth reporting either. `std::fs::remove_dir` itself
8189/// already refuses a non-empty directory, so the emptiness check below is
8190/// belt, not suspenders - it is what keeps this from ever attempting the
8191/// removal in the case that matters, rather than trusting `remove_dir`'s
8192/// error path to have no side effects if it ever changed.
8193fn remove_if_empty(dir: &Path) {
8194    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
8195        std::fs::remove_dir(dir).ok();
8196    }
8197}
8198
8199/// Severity of the worst open finding in the last review round, for reporting.
8200pub fn worst_open(state: &RunState) -> Option<Severity> {
8201    state
8202        .reviews
8203        .last()?
8204        .reviews
8205        .iter()
8206        .flat_map(|r| r.findings.iter())
8207        .map(|f| f.severity)
8208        .max()
8209}
8210
8211#[cfg(test)]
8212mod tests {
8213    #[test]
8214    fn pr_merge_plan_creates_adopts_or_stops() {
8215        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
8216        assert_eq!(
8217            pr_merge_plan(Ok(land::OpenPr::One {
8218                url: "u".into(),
8219                title: "t".into()
8220            })),
8221            PrPlan::Adopt {
8222                url: "u".into(),
8223                title: "t".into()
8224            }
8225        );
8226        let PrPlan::Stop(many) =
8227            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
8228        else {
8229            panic!("many must stop");
8230        };
8231        assert!(many.contains('a') && many.contains('b'));
8232        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
8233            panic!("a failed lookup must stop");
8234        };
8235        assert!(err.contains("bad token"));
8236    }
8237
8238    use super::*;
8239    use crate::run::GateStatus;
8240    use std::collections::BTreeMap;
8241    use std::time::Duration;
8242
8243    fn conductor() -> AgentSpec {
8244        AgentSpec {
8245            id: "conductor".to_owned(),
8246            kind: crate::config::AgentKind::Command,
8247            model: None,
8248            command: vec!["true".to_owned()],
8249            extra_args: Vec::new(),
8250            env: BTreeMap::new(),
8251            prompt_delivery: None,
8252        }
8253    }
8254
8255    fn spec(id: &str) -> AgentSpec {
8256        AgentSpec {
8257            id: id.to_owned(),
8258            kind: crate::config::AgentKind::Command,
8259            model: None,
8260            command: vec!["true".to_owned()],
8261            extra_args: Vec::new(),
8262            env: BTreeMap::new(),
8263            prompt_delivery: None,
8264        }
8265    }
8266
8267    fn ids(xs: &[&str]) -> BTreeSet<String> {
8268        xs.iter().map(|s| (*s).to_owned()).collect()
8269    }
8270
8271    #[test]
8272    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
8273        let roster = [spec("a"), spec("b"), spec("c")];
8274        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
8275        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
8276    }
8277
8278    #[test]
8279    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
8280        let roster = [spec("a"), spec("b"), spec("c")];
8281        let failed = ids(&["a", "b", "c"]);
8282        // Rescue looks at the whole roster, once per id, then runs out.
8283        let mut tried = ids(&["b"]);
8284        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8285        assert_eq!(first.id, "a");
8286        tried.insert(first.id.clone());
8287        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
8288        assert_eq!(second.id, "c");
8289        tried.insert(second.id.clone());
8290        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
8291    }
8292
8293    #[test]
8294    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
8295        let roster = [spec("a"), spec("b")];
8296        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
8297        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8298    }
8299
8300    #[test]
8301    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
8302        let roster = [spec("a"), spec("b"), spec("c")];
8303        let h = SeatHistory {
8304            failed: ids(&["a"]),
8305            last_ok: Some("b".to_owned()),
8306            last_fail: None,
8307        };
8308        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
8309        // A last answerer that left the roster, or later failed, is ignored.
8310        let gone = SeatHistory {
8311            last_ok: Some("zzz".to_owned()),
8312            ..h.clone()
8313        };
8314        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
8315        let failed = SeatHistory {
8316            failed: ids(&["a", "b"]),
8317            last_ok: Some("b".to_owned()),
8318            last_fail: None,
8319        };
8320        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
8321    }
8322
8323    #[test]
8324    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
8325        let first = SeatState::new("review-1", "alpha", 7);
8326        let next = handover_seat("review-1", "gamma", 7);
8327        assert_ne!(first.claude_session, next.claude_session);
8328    }
8329
8330    #[test]
8331    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
8332        let mut state = state_with_summary("x", "y");
8333        let a = handover_seat("review-1", "beta", state.next_seat_seed());
8334        let b = handover_seat("review-1", "beta", state.next_seat_seed());
8335        assert_ne!(a.claude_session, b.claude_session);
8336    }
8337
8338    #[test]
8339    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
8340        let roster = [spec("a"), spec("b")];
8341        let h = SeatHistory {
8342            failed: ids(&["a", "b"]),
8343            ..SeatHistory::default()
8344        };
8345        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
8346        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
8347        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
8348    }
8349
8350    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
8351    // fallback loop depends on to terminate: it must walk forward from the
8352    // seat's own position, never restart at the front of the roster, and it
8353    // must never hand back an id already tried, however many times that id
8354    // happens to appear.
8355
8356    #[test]
8357    fn failure_signature_ignores_numbers_and_paths() {
8358        assert_eq!(
8359            failure_signature("exited with Some(2) and no usable output"),
8360            failure_signature("exited with Some(137) and no usable output")
8361        );
8362        assert_eq!(
8363            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
8364            failure_signature("cannot open /var/x.txt: denied")
8365        );
8366        assert_ne!(failure_signature("boom"), failure_signature("bang"));
8367    }
8368
8369    #[test]
8370    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
8371        let other = FailClass::Other("x".into());
8372        assert!(should_hand_over(None, &FailClass::Quota));
8373        assert!(should_hand_over(Some(&other), &FailClass::Quota));
8374        assert!(should_hand_over(
8375            Some(&FailClass::Timeout),
8376            &FailClass::Timeout
8377        ));
8378        assert!(should_hand_over(None, &other));
8379        assert!(!should_hand_over(Some(&other), &other));
8380        assert!(should_hand_over(
8381            Some(&other),
8382            &FailClass::Other("y".into())
8383        ));
8384        // A quota or timeout in between ends the run of identical failures.
8385        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
8386        assert!(should_hand_over(Some(&FailClass::Quota), &other));
8387    }
8388
8389    #[test]
8390    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
8391        let a = SeatState::new("judge-1", "alpha", 7);
8392        let b = handover_seat("judge-1", "beta", 7);
8393        assert_ne!(a.claude_session, b.claude_session);
8394        assert_eq!(b.turns, 0);
8395    }
8396
8397    #[test]
8398    fn a_timeout_is_classified_apart_from_other_failures() {
8399        assert_eq!(
8400            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
8401            Some(FailClass::Timeout)
8402        );
8403        assert!(matches!(
8404            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
8405            Some(FailClass::Other(_))
8406        ));
8407    }
8408
8409    #[test]
8410    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
8411        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8412        let tried = BTreeSet::from(["beta".to_owned()]);
8413        // beta sits at index 1; the next candidate is gamma, never alpha —
8414        // which is very likely a different candidate slot's own agent.
8415        let next = next_untried_in_roster(&roster, 1, &tried);
8416        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
8417    }
8418
8419    #[test]
8420    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
8421        let roster = vec![spec("alpha"), spec("beta")];
8422        let tried = BTreeSet::from(["beta".to_owned()]);
8423        // beta is the roster's last entry: nothing follows it, and alpha —
8424        // earlier in the roster, almost certainly a different candidate
8425        // slot's own agent — must not be reached by wrapping back to it.
8426        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8427    }
8428
8429    #[test]
8430    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
8431        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
8432        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
8433        // beta (index 1) and gamma (index 2, the only entry after it) have
8434        // both been tried; alpha (index 0) never has, but it comes before
8435        // beta's own position, so there is nothing further for this seat.
8436        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
8437    }
8438
8439    #[test]
8440    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
8441        let roster = vec![spec("a"), spec("a"), spec("b")];
8442        let tried = BTreeSet::from(["a".to_owned()]);
8443        let next = next_untried_in_roster(&roster, 0, &tried);
8444        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
8445    }
8446
8447    #[test]
8448    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
8449        let roster = vec![spec("a"), spec("b")];
8450        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
8451        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
8452    }
8453
8454    #[test]
8455    fn remove_if_empty_only_ever_takes_a_bare_directory() {
8456        let dir = tempfile::tempdir().unwrap();
8457        let bay = dir.path().join("ffff");
8458
8459        // Not there yet: nothing to do, nothing to panic on.
8460        remove_if_empty(&bay);
8461        assert!(!bay.exists());
8462
8463        // Something still inside - the winner's worktree, or a stray file -
8464        // keeps the directory standing.
8465        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
8466        remove_if_empty(&bay);
8467        assert!(bay.exists(), "non-empty directory must survive");
8468
8469        // Once the last entry is gone, so is the directory itself.
8470        std::fs::remove_dir(bay.join("cand-A")).unwrap();
8471        remove_if_empty(&bay);
8472        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
8473    }
8474
8475    // `round_is_clean` is the exact decision this task fixed: a round with a
8476    // seat that never answered must not read the same as a round every seat
8477    // actually reviewed. These are deterministic and process-free by design —
8478    // the equivalent end-to-end check (a real reviewer timing out under a
8479    // live graph run) is a genuine race against wall-clock contention, and a
8480    // spawn slow enough to blow even a generous budget under a loaded test
8481    // run must not turn this specific regression check flaky.
8482
8483    #[test]
8484    fn a_full_panel_that_found_nothing_is_clean() {
8485        assert!(round_is_clean(
8486            0,
8487            true,
8488            2,
8489            2,
8490            0,
8491            IncompleteReviewPolicy::Block
8492        ));
8493    }
8494
8495    #[test]
8496    fn a_missing_seat_is_never_clean_under_the_default_policy() {
8497        assert!(!round_is_clean(
8498            0,
8499            true,
8500            1,
8501            2,
8502            0,
8503            IncompleteReviewPolicy::Block
8504        ));
8505    }
8506
8507    #[test]
8508    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
8509        assert!(!round_is_clean(
8510            1,
8511            true,
8512            1,
8513            2,
8514            0,
8515            IncompleteReviewPolicy::Warn
8516        ));
8517    }
8518
8519    #[test]
8520    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
8521        assert!(round_is_clean(
8522            0,
8523            true,
8524            1,
8525            2,
8526            0,
8527            IncompleteReviewPolicy::Warn
8528        ));
8529    }
8530
8531    #[test]
8532    fn a_full_panel_with_an_open_finding_is_not_clean() {
8533        assert!(!round_is_clean(
8534            1,
8535            true,
8536            2,
8537            2,
8538            0,
8539            IncompleteReviewPolicy::Block
8540        ));
8541    }
8542
8543    #[test]
8544    fn a_full_panel_with_a_red_e2e_is_not_clean() {
8545        assert!(!round_is_clean(
8546            0,
8547            false,
8548            2,
8549            2,
8550            0,
8551            IncompleteReviewPolicy::Block
8552        ));
8553    }
8554
8555    // The stall this task closes: under the default `block` policy, a seat
8556    // missing only because it was rate limited must not force a wait for a
8557    // session limit that will not lift by the next round. `round_is_clean`
8558    // is where that quorum carve-out lives; the review loop around it never
8559    // changes what a reviewer's vote or a finding's severity means.
8560
8561    #[test]
8562    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
8563        // 1 of 2 answered, and the one missing was quota'd — the exact
8564        // "review-2 rate limited (quota)" shape from the field report.
8565        assert!(round_is_clean(
8566            0,
8567            true,
8568            1,
8569            2,
8570            1,
8571            IncompleteReviewPolicy::Block
8572        ));
8573    }
8574
8575    #[test]
8576    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
8577        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
8578        // not a quota loss (`quota_missing` stays 0) — worth another try.
8579        assert!(!round_is_clean(
8580            0,
8581            true,
8582            1,
8583            2,
8584            0,
8585            IncompleteReviewPolicy::Block
8586        ));
8587    }
8588
8589    #[test]
8590    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
8591        assert!(!round_is_clean(
8592            1,
8593            true,
8594            1,
8595            2,
8596            1,
8597            IncompleteReviewPolicy::Block
8598        ));
8599        assert!(!round_is_clean(
8600            0,
8601            false,
8602            1,
8603            2,
8604            1,
8605            IncompleteReviewPolicy::Block
8606        ));
8607    }
8608
8609    #[test]
8610    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
8611        // Every seat quota'd, nobody answered: there is no panel to decide
8612        // on, so this must fall through to the existing block-and-retry
8613        // fallback rather than call an unreviewed patch clean.
8614        assert!(!round_is_clean(
8615            0,
8616            true,
8617            0,
8618            2,
8619            2,
8620            IncompleteReviewPolicy::Block
8621        ));
8622    }
8623
8624    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
8625        CommandOutcome {
8626            command: "test".to_owned(),
8627            code,
8628            output_tail: String::new(),
8629            duration_ms: 0,
8630            resource_blocked,
8631        }
8632    }
8633
8634    #[test]
8635    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
8636        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
8637        assert!(
8638            !verify_inconclusive(&[outcome(Some(1), false)]),
8639            "an ordinary failure is still evidence about the patch"
8640        );
8641        assert!(verify_inconclusive(&[outcome(None, true)]));
8642        assert!(
8643            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
8644            "one inconclusive outcome taints the whole batch"
8645        );
8646        assert!(!verify_inconclusive(&[]));
8647    }
8648
8649    #[tokio::test]
8650    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
8651        // Alive for the first two checks, then dead - confirms the loop
8652        // actually re-polls rather than deciding once and sleeping out the
8653        // ceiling regardless.
8654        let calls = std::sync::atomic::AtomicUsize::new(0);
8655        let started = Instant::now();
8656        wait_for_pids_with(
8657            &[123],
8658            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
8659            Duration::from_millis(5),
8660            Duration::from_secs(5),
8661        )
8662        .await;
8663        assert!(
8664            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
8665            "must keep checking rather than deciding on the first answer"
8666        );
8667        assert!(
8668            started.elapsed() < Duration::from_secs(1),
8669            "must return the moment it is confirmed dead, not wait out the ceiling"
8670        );
8671    }
8672
8673    #[tokio::test]
8674    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
8675        let started = Instant::now();
8676        wait_for_pids_with(
8677            &[123],
8678            |_| true, // never reports dead
8679            Duration::from_millis(5),
8680            Duration::from_millis(30),
8681        )
8682        .await;
8683        let elapsed = started.elapsed();
8684        assert!(
8685            elapsed >= Duration::from_millis(30),
8686            "must not give up before its own ceiling: {elapsed:?}"
8687        );
8688        assert!(
8689            elapsed < Duration::from_secs(1),
8690            "must not wait past its own ceiling either: {elapsed:?}"
8691        );
8692    }
8693
8694    #[tokio::test]
8695    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
8696        let started = Instant::now();
8697        wait_for_pids_with(
8698            &[],
8699            |_| true,
8700            Duration::from_secs(5),
8701            Duration::from_secs(5),
8702        )
8703        .await;
8704        assert!(
8705            started.elapsed() < Duration::from_millis(200),
8706            "an empty pid list has nothing to confirm"
8707        );
8708    }
8709
8710    // `review_conclusion` is the exact decision the review hand-off task
8711    // fixed: a round budget spent (or a tree that stopped moving) must not
8712    // collapse into `Blocked` regardless of what verification actually
8713    // said. Deterministic and process-free for the same reason the
8714    // `round_is_clean` family above is.
8715    fn review_round(
8716        clean: bool,
8717        blocking: usize,
8718        answered: usize,
8719        expected: usize,
8720        progressed: bool,
8721        e2e_ok: bool,
8722    ) -> ReviewRound {
8723        ReviewRound {
8724            round: 1,
8725            head: "h".to_owned(),
8726            verified_head: None,
8727            verified_at: None,
8728            reviews: Vec::new(),
8729            e2e: vec![CommandOutcome {
8730                command: "test".to_owned(),
8731                code: Some(if e2e_ok { 0 } else { 1 }),
8732                output_tail: String::new(),
8733                duration_ms: 0,
8734                resource_blocked: false,
8735            }],
8736            verify_retried: false,
8737            e2e_deferred: false,
8738            e2e_defer_reason: None,
8739            fix: None,
8740            blocking,
8741            answered,
8742            expected,
8743            clean,
8744            progressed,
8745            vote_split: false,
8746            reconsideration: Vec::new(),
8747            verdict: None,
8748        }
8749    }
8750
8751    #[test]
8752    fn review_conclusion_is_none_when_nothing_has_run() {
8753        assert_eq!(review_conclusion(&[], 3), None);
8754    }
8755
8756    #[test]
8757    fn review_conclusion_is_none_while_rounds_remain() {
8758        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
8759        assert_eq!(review_conclusion(&rounds, 3), None);
8760    }
8761
8762    #[test]
8763    fn review_conclusion_is_gating_once_a_round_is_clean() {
8764        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
8765        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
8766    }
8767
8768    #[test]
8769    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
8770        let rounds = vec![
8771            review_round(false, 1, 2, 2, true, true),
8772            review_round(false, 1, 2, 2, true, true),
8773        ];
8774        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
8775    }
8776
8777    #[test]
8778    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
8779        let rounds = vec![
8780            review_round(false, 1, 2, 2, true, true),
8781            review_round(false, 1, 2, 2, true, false),
8782        ];
8783        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
8784    }
8785
8786    #[test]
8787    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
8788        // Magi never got a command to run against this round's own head — a
8789        // resource-blocked attempt, not a red one — so this must never
8790        // settle on `Blocked` the way a genuine e2e failure would. `None`
8791        // here is what tells `Runner::review_loop` to retry the check
8792        // itself rather than trust this cheap recomputation with a verdict
8793        // it cannot actually produce.
8794        let mut blocked = review_round(false, 1, 2, 2, true, false);
8795        blocked.e2e[0].resource_blocked = true;
8796        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
8797        assert_eq!(review_conclusion(&rounds, 2), None);
8798    }
8799
8800    #[test]
8801    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
8802        // Missing input, not a verified tree — never a hand-off candidate.
8803        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
8804        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
8805    }
8806
8807    #[test]
8808    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
8809        let rounds = vec![
8810            review_round(false, 1, 2, 2, false, true),
8811            review_round(false, 1, 2, 2, false, true),
8812        ];
8813        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
8814    }
8815
8816    fn secs(n: u64) -> Duration {
8817        Duration::from_secs(n)
8818    }
8819
8820    /// A throwaway repo with one commit on `main`, for tests that need `merge`
8821    /// to make real (and, if it runs at all, real*ly fail*) git calls.
8822    fn init_repo(dir: &Path) {
8823        let run = |args: &[&str]| {
8824            let out = std::process::Command::new("git")
8825                .args(args)
8826                .current_dir(dir)
8827                .quiet()
8828                .output()
8829                .expect("spawn git");
8830            assert!(
8831                out.status.success(),
8832                "git {args:?} failed: {}",
8833                String::from_utf8_lossy(&out.stderr)
8834            );
8835        };
8836        run(&["init", "-b", "main"]);
8837        run(&["config", "user.name", "magi test"]);
8838        run(&["config", "user.email", "magi@example.com"]);
8839        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
8840        run(&["add", "-A"]);
8841        run(&["commit", "-m", "init"]);
8842    }
8843
8844    // `settle_questions` is what closes the ghost the phone showed: a run's
8845    // seat asked something, the run then ended, and nothing was left to
8846    // abandon the question it left `open`. `HOME` is a process-wide
8847    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
8848    // first time it runs in the binary — every test below still reaches the
8849    // same directory whichever call won, and each gets its own run id from
8850    // `RunState::new`, so they never collide there.
8851    fn ask_test_home() {
8852        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
8853    }
8854
8855    /// A minimal, git-free `Runner` at a given status — `settle_questions`
8856    /// reads nothing else off it.
8857    fn runner_at(status: RunStatus) -> Runner {
8858        let mut state = RunState::new(
8859            PathBuf::from("/nonexistent/repo"),
8860            "main".to_owned(),
8861            "deadbeef".to_owned(),
8862            "task".to_owned(),
8863            Config::default(),
8864        );
8865        state.status = status;
8866        Runner {
8867            state,
8868            roles: ResolvedRoles {
8869                implementers: Vec::new(),
8870                judges: Vec::new(),
8871                reviewers: Vec::new(),
8872                fixer: None,
8873                conductor: conductor(),
8874                implementer_roster: Vec::new(),
8875                judge_roster: Vec::new(),
8876                reviewer_roster: Vec::new(),
8877            },
8878            sem: Arc::new(Semaphore::new(1)),
8879            pause: Pause::new(),
8880            interrupt: Pause::new(),
8881        }
8882    }
8883
8884    /// `park_here` folding in the reason `Pause::park_because` recorded -
8885    /// this is what lets an operator reading a run's events tell an
8886    /// interrupt-driven park from an ordinary shutdown park.
8887    #[test]
8888    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
8889        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8890        let mut runner = runner_at(RunStatus::Implementing);
8891        let interrupt = Pause::new();
8892        runner.watch_interrupt(interrupt.clone());
8893
8894        interrupt.park_because("task a1b2 asked to run first");
8895
8896        assert!(runner.park_here().expect("park_here"));
8897        assert!(runner.state.parked);
8898        let last = runner.state.events.last().expect("a park event");
8899        assert_eq!(last.node, "park");
8900        assert!(
8901            last.message.contains("task a1b2 asked to run first"),
8902            "expected the interrupt reason in {:?}",
8903            last.message
8904        );
8905    }
8906
8907    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
8908    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
8909    /// cleared) must not make a *different* run - one only watching its own,
8910    /// unshared interrupt `Pause` - see itself as parked. If a future change
8911    /// ever collapsed these back into one handle, the interrupt scheduler
8912    /// would park every run for the rest of the daemon's life, not just the
8913    /// one it meant to interrupt.
8914    #[test]
8915    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
8916        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8917        let mut runner = runner_at(RunStatus::Implementing);
8918        let shutdown = Pause::new();
8919        runner.on_pause(shutdown.clone());
8920        let interrupt = Pause::new();
8921        runner.watch_interrupt(interrupt.clone());
8922
8923        // Nobody has asked for anything yet.
8924        assert!(!runner.park_here().expect("park_here"));
8925        assert!(!runner.state.parked);
8926
8927        // Only the interrupt handle fires; the shutdown handle stays clear.
8928        interrupt.park_because("test");
8929        assert!(!shutdown.parked());
8930        assert!(runner.park_here().expect("park_here"));
8931    }
8932
8933    /// The property every prior attempt at this feature failed to pin down:
8934    /// asking a run to park while one of its nodes has a real, in-flight
8935    /// async operation running (an agent call, in production) must not cut
8936    /// that operation short. `park_here` is only ever consulted *between*
8937    /// `execute`'s node calls - see its own doc - so nothing inside a node
8938    /// can observe a park request until the node itself returns. This proves
8939    /// that structurally, with real `tokio` concurrency and a channel
8940    /// handshake (never a sleep, which would only prove "usually", not
8941    /// "cannot"): the "node" below reports that it has genuinely started,
8942    /// and only then is the park requested; the node still has to be told to
8943    /// finish before `park_here` is ever called, exactly mirroring every
8944    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
8945    /// in `execute`.
8946    #[tokio::test]
8947    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
8948        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8949        let mut runner = runner_at(RunStatus::Implementing);
8950        let interrupt = Pause::new();
8951        runner.watch_interrupt(interrupt.clone());
8952
8953        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
8954        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
8955
8956        // Stands in for one node's in-flight agent call: it proves it has
8957        // genuinely started, then blocks - exactly as a spawned CLI process
8958        // does - until told to finish.
8959        let node = async move {
8960            started_tx.send(()).expect("send started");
8961            finish_rx.await.expect("recv finish");
8962            "node finished"
8963        };
8964
8965        let interrupter = async move {
8966            started_rx.await.expect("recv started");
8967            // The call is now genuinely in flight. Ask it to park.
8968            interrupt.park_because("higher-priority task waiting");
8969            // Nothing the node does can observe this yet - there is no
8970            // check inside it, by construction - so let the executor run
8971            // anything pending and then let the node finish on its own.
8972            tokio::task::yield_now().await;
8973            finish_tx.send(()).expect("send finish");
8974        };
8975
8976        let (node_result, ()) = tokio::join!(node, interrupter);
8977        assert_eq!(
8978            node_result, "node finished",
8979            "the in-flight call ran to completion"
8980        );
8981
8982        // Only now, at the boundary the real `execute` would check right
8983        // after this node, does the park take effect.
8984        assert!(runner.park_here().expect("park_here"));
8985        assert!(runner.state.parked);
8986    }
8987
8988    /// A run parked mid-competition carries every field it had accumulated
8989    /// through the exact same disk round-trip an ordinary resume uses -
8990    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
8991    /// Nothing about parking for an interrupt is a special case of that path;
8992    /// this is what proves it rather than assuming it.
8993    #[test]
8994    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
8995        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
8996        let mut runner = runner_at(RunStatus::Judging);
8997        // `Runner::resume` re-resolves roles from the saved config, which
8998        // refuses an empty roster - give it the same minimal one `conductor`
8999        // itself uses.
9000        runner.state.config.agents = vec![conductor()];
9001        runner.state.candidates = vec![Candidate {
9002            index: 0,
9003            label: 'A',
9004            agent: "alpha".to_owned(),
9005            branch: "magi/x/A".to_owned(),
9006            worktree: PathBuf::from("/nonexistent/worktree"),
9007            summary: "did the thing".to_owned(),
9008            stat: "1 file changed".to_owned(),
9009            files: 1,
9010            commits: 1,
9011            empty: false,
9012            failed: None,
9013            verified_noop: None,
9014            duration_ms: 1234,
9015            folded: false,
9016        }];
9017        let run_id = runner.state.id.clone();
9018
9019        let interrupt = Pause::new();
9020        runner.watch_interrupt(interrupt.clone());
9021        interrupt.park_because("task c3d4 asked to run first");
9022        assert!(runner.park_here().expect("park_here"));
9023
9024        let resumed = Runner::resume(&run_id).expect("resume");
9025        assert_eq!(resumed.state.candidates.len(), 1);
9026        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
9027        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
9028        assert_eq!(resumed.state.status, runner.state.status);
9029        assert!(
9030            resumed.state.parked,
9031            "still parked until `execute` actually walks the graph again"
9032        );
9033        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
9034    }
9035
9036    /// A fresh open question on `run`, stored and handed back for assertions.
9037    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
9038        let mut q = ask::Question::new(
9039            run.to_owned(),
9040            "implement".to_owned(),
9041            "impl-A".to_owned(),
9042            "Which storage backend should the cache use?".to_owned(),
9043            String::new(),
9044            vec!["SQLite".to_owned(), "Redis".to_owned()],
9045        );
9046        store.put(&mut q).unwrap();
9047        q
9048    }
9049
9050    #[test]
9051    fn a_failed_runs_open_question_is_abandoned() {
9052        ask_test_home();
9053        let store = ask::Questions::open();
9054        let mut runner = runner_at(RunStatus::Failed);
9055        let run = runner.state.id.clone();
9056        let q = ask_open_question(&store, &run);
9057
9058        runner.settle_questions();
9059
9060        let back = store.get(&q.id).unwrap();
9061        assert!(
9062            !back.status.open(),
9063            "the seat that asked died with the run; nobody is left to read an answer"
9064        );
9065        assert!(
9066            back.detail.contains(&run) && back.detail.contains("failed"),
9067            "the reason names what the run became, not just that it is gone: {}",
9068            back.detail
9069        );
9070    }
9071
9072    #[test]
9073    fn a_merged_runs_open_question_is_abandoned_too() {
9074        ask_test_home();
9075        let store = ask::Questions::open();
9076        // A run that finishes cleanly still leaves nobody to read an answer -
9077        // this is not only a failure-path cleanup.
9078        for status in [RunStatus::Merged, RunStatus::Ready] {
9079            let mut runner = runner_at(status);
9080            let run = runner.state.id.clone();
9081            let q = ask_open_question(&store, &run);
9082
9083            runner.settle_questions();
9084
9085            let back = store.get(&q.id).unwrap();
9086            assert!(
9087                !back.status.open(),
9088                "{status:?} run's question must not outlive the run"
9089            );
9090        }
9091    }
9092
9093    #[test]
9094    fn a_still_resumable_runs_open_question_is_left_alone() {
9095        ask_test_home();
9096        let store = ask::Questions::open();
9097        // `Blocked` and `Stalled` can still be resumed — the candidates, the
9098        // review round and the seat sessions are all still on disk — so a
9099        // question asked mid-round may yet get a real answer from a real
9100        // resume. Sweeping it here would be exactly the failure mode this
9101        // whole feature exists to avoid on the other side.
9102        for status in [RunStatus::Blocked, RunStatus::Stalled] {
9103            let mut runner = runner_at(status);
9104            let run = runner.state.id.clone();
9105            let q = ask_open_question(&store, &run);
9106
9107            runner.settle_questions();
9108
9109            let back = store.get(&q.id).unwrap();
9110            assert!(
9111                back.status.open(),
9112                "{status:?} is still alive; the question must still be waiting"
9113            );
9114        }
9115    }
9116
9117    #[test]
9118    fn settle_questions_never_touches_an_already_answered_question() {
9119        ask_test_home();
9120        let store = ask::Questions::open();
9121        let mut runner = runner_at(RunStatus::Failed);
9122        let run = runner.state.id.clone();
9123        let mut q = ask_open_question(&store, &run);
9124        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
9125            .unwrap();
9126        store.put(&mut q).unwrap();
9127
9128        // Called twice, the way a crash-recovered daemon reclaim and the
9129        // graph's own cleanup both can for the same run — `abandon_for_run`
9130        // only ever touches what is still open, so this must be inert both
9131        // times, not merely the second.
9132        runner.settle_questions();
9133        runner.settle_questions();
9134
9135        let back = store.get(&q.id).unwrap();
9136        assert_eq!(
9137            back.status,
9138            ask::QuestionStatus::Answered,
9139            "a real answer is a decision on record, never overwritten by a sweep"
9140        );
9141    }
9142
9143    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
9144    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
9145    /// without merging, whose winner is still the operator's answer to read.
9146    /// Nothing previously called `fold_run` itself with a real `tally`, so
9147    /// this is the first test to pin down the one distinction the whole
9148    /// automatic-fold feature depends on: the winner's worktree and branch
9149    /// must survive, everything else sharing the run's worktree bay - a
9150    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
9151    /// second sweep treats every non-winner directory under the bay alike -
9152    /// must not.
9153    #[tokio::test]
9154    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
9155        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
9156        let tmp = tempfile::tempdir().expect("tempdir");
9157        let repo = tmp.path().join("repo");
9158        std::fs::create_dir_all(&repo).unwrap();
9159        init_repo(&repo);
9160
9161        let mut config = Config::default();
9162        config.graph.worktree_root = Some(tmp.path().join("wt"));
9163
9164        let mut state = RunState::new(
9165            repo.clone(),
9166            "main".to_owned(),
9167            "deadbeef".to_owned(),
9168            "task".to_owned(),
9169            config,
9170        );
9171        let root = state.worktree_root();
9172        let wt_a = root.join("cand-A");
9173        let wt_b = root.join("cand-B");
9174        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
9175            .await
9176            .expect("worktree A");
9177        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
9178            .await
9179            .expect("worktree B");
9180
9181        state.candidates = vec![
9182            Candidate {
9183                index: 0,
9184                label: 'A',
9185                agent: "alpha".to_owned(),
9186                branch: "magi/x/A".to_owned(),
9187                worktree: wt_a.clone(),
9188                summary: String::new(),
9189                stat: String::new(),
9190                files: 0,
9191                commits: 0,
9192                empty: false,
9193                failed: None,
9194                verified_noop: None,
9195                duration_ms: 0,
9196                folded: false,
9197            },
9198            Candidate {
9199                index: 1,
9200                label: 'B',
9201                agent: "beta".to_owned(),
9202                branch: "magi/x/B".to_owned(),
9203                worktree: wt_b.clone(),
9204                summary: String::new(),
9205                stat: String::new(),
9206                files: 0,
9207                commits: 0,
9208                empty: false,
9209                failed: None,
9210                verified_noop: None,
9211                duration_ms: 0,
9212                folded: false,
9213            },
9214        ];
9215        state.tally = Some(Tally {
9216            first_choice: BTreeMap::from([('A', 1)]),
9217            borda: BTreeMap::new(),
9218            winner: 'A',
9219            rankings: 1,
9220            unanimous_initial: true,
9221            deliberated: false,
9222            changed_votes: 0,
9223            unanimous_final: true,
9224            tie_break: None,
9225            judges: 1,
9226            present: 1,
9227            quorum: 1,
9228            met_quorum: true,
9229            uncontested: None,
9230        });
9231        state.status = RunStatus::Ready;
9232
9233        fold_run(&mut state, false, &crate::run::home())
9234            .await
9235            .expect("fold_run");
9236
9237        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
9238        assert!(
9239            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9240            "the unmerged winner's branch survives"
9241        );
9242        assert!(
9243            !state.candidates[0].folded,
9244            "the winner is not marked folded"
9245        );
9246
9247        assert!(!wt_b.exists(), "the loser's worktree is removed");
9248        assert!(
9249            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
9250            "the loser's branch is removed"
9251        );
9252        assert!(state.candidates[1].folded, "the loser is marked folded");
9253    }
9254
9255    /// A branch handed to a later run is that run's (and its pull request's):
9256    /// folding the run that released it must not delete it.
9257    #[tokio::test]
9258    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
9259        let tmp = tempfile::tempdir().expect("tempdir");
9260        let repo = tmp.path().join("repo");
9261        std::fs::create_dir_all(&repo).unwrap();
9262        init_repo(&repo);
9263        let home = tmp.path().join("home");
9264
9265        let mut config = Config::default();
9266        config.graph.worktree_root = Some(tmp.path().join("wt"));
9267        let mut state = RunState::new(
9268            repo.clone(),
9269            "main".to_owned(),
9270            "deadbeef".to_owned(),
9271            "task".to_owned(),
9272            config,
9273        );
9274        // The worktree is already gone (released); the branch survives.
9275        git::git(&repo, &["branch", "magi/x/A", "main"])
9276            .await
9277            .expect("branch");
9278        state.candidates = vec![Candidate {
9279            index: 0,
9280            label: 'A',
9281            agent: "alpha".to_owned(),
9282            branch: "magi/x/A".to_owned(),
9283            worktree: state.worktree_root().join("cand-A"),
9284            summary: String::new(),
9285            stat: String::new(),
9286            files: 0,
9287            commits: 0,
9288            empty: false,
9289            failed: None,
9290            verified_noop: None,
9291            duration_ms: 0,
9292            folded: true,
9293        }];
9294        state.released_to = Some("20260901-000000-new1".to_owned());
9295        state.released_branches = vec!["magi/x/A".to_owned()];
9296
9297        fold_run(&mut state, true, &home).await.expect("fold_run");
9298
9299        assert!(
9300            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
9301            "the handed-over branch survives a fold"
9302        );
9303    }
9304
9305    /// A winner with nothing ahead of the base is caught before `gh` is ever
9306    /// asked for a pull request, and the message carries what the task's
9307    /// references resolved to.
9308    #[tokio::test]
9309    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
9310        let tmp = tempfile::tempdir().expect("tempdir");
9311        let repo = tmp.path().join("repo");
9312        std::fs::create_dir_all(&repo).unwrap();
9313        init_repo(&repo);
9314        let run = |args: &[&str]| {
9315            let out = std::process::Command::new("git")
9316                .quiet()
9317                .args(args)
9318                .current_dir(&repo)
9319                .output()
9320                .expect("spawn git");
9321            assert!(out.status.success(), "git {args:?}");
9322        };
9323        run(&["branch", "magi/x/A"]);
9324        run(&["checkout", "-q", "-b", "magi/x/B"]);
9325        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
9326        run(&["add", "-A"]);
9327        run(&["commit", "-q", "-m", "work"]);
9328        run(&["checkout", "-q", "main"]);
9329
9330        let mut state = RunState::new(
9331            repo.clone(),
9332            "main".to_owned(),
9333            "deadbeef".to_owned(),
9334            "task".to_owned(),
9335            Config::default(),
9336        );
9337        state.seeds = vec![refs::Seed {
9338            token: "magi/27b2/A".to_owned(),
9339            kind: refs::SeedKind::Unresolved,
9340            sha: String::new(),
9341            branch: true,
9342            detail: "no branch or commit named magi/27b2/A".to_owned(),
9343        }];
9344
9345        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
9346        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
9347        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
9348        let detail = empty_candidate_detail(&state, "main");
9349        assert!(detail.starts_with("empty candidate"), "{detail}");
9350        assert!(detail.contains("magi/27b2/A"), "{detail}");
9351    }
9352
9353    /// `status == Ready` used to be read as "this is the harmless
9354    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
9355    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
9356    /// run's PR was closed without merging — and reentering `merge` with
9357    /// `mode` still `Pr` does not know the difference, so it pushed and
9358    /// opened a second pull request. `mode == Local` reproduces the same
9359    /// blind spot without a network call: reentry must not attempt another
9360    /// git merge once this node has already recorded an outcome.
9361    #[tokio::test]
9362    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
9363        let tmp = tempfile::tempdir().expect("tempdir");
9364        let repo = tmp.path().join("repo");
9365        std::fs::create_dir_all(&repo).unwrap();
9366        init_repo(&repo);
9367
9368        let mut config = Config::default();
9369        config.merge.mode = MergeMode::Local;
9370
9371        let mut state = RunState::new(
9372            repo.clone(),
9373            "main".to_owned(),
9374            "deadbeef".to_owned(),
9375            "task".to_owned(),
9376            config,
9377        );
9378        state.candidates = vec![Candidate {
9379            index: 0,
9380            label: 'A',
9381            agent: "alpha".to_owned(),
9382            branch: "does-not-exist".to_owned(),
9383            worktree: repo.clone(),
9384            summary: String::new(),
9385            stat: String::new(),
9386            files: 0,
9387            commits: 0,
9388            empty: false,
9389            failed: None,
9390            verified_noop: None,
9391            duration_ms: 0,
9392            folded: false,
9393        }];
9394        state.tally = Some(Tally {
9395            first_choice: BTreeMap::from([('A', 1)]),
9396            borda: BTreeMap::new(),
9397            winner: 'A',
9398            rankings: 1,
9399            unanimous_initial: true,
9400            deliberated: false,
9401            changed_votes: 0,
9402            unanimous_final: true,
9403            tie_break: None,
9404            judges: 0,
9405            present: 0,
9406            quorum: 0,
9407            met_quorum: true,
9408            uncontested: Some("only candidate A produced a change".to_owned()),
9409        });
9410        state.reviews = vec![ReviewRound {
9411            round: 1,
9412            head: "deadbeef".to_owned(),
9413            verified_head: None,
9414            verified_at: None,
9415            reviews: Vec::new(),
9416            e2e: Vec::new(),
9417            fix: None,
9418            blocking: 0,
9419            answered: 0,
9420            expected: 0,
9421            clean: true,
9422            verify_retried: false,
9423            e2e_deferred: false,
9424            e2e_defer_reason: None,
9425            progressed: false,
9426            vote_split: false,
9427            reconsideration: Vec::new(),
9428            verdict: None,
9429        }];
9430        state.gate = vec![CommandOutcome {
9431            command: "test".to_owned(),
9432            code: Some(0),
9433            output_tail: String::new(),
9434            duration_ms: 0,
9435            resource_blocked: false,
9436        }];
9437        state.gate_ran = true;
9438        // Reached its conclusion already — e.g. `land` closing the PR without
9439        // merging it, which (like the honest `MergeMode::None` path) leaves
9440        // `status` at `Ready`. The recorded outcome is what actually marks
9441        // this node done.
9442        state.status = RunStatus::Ready;
9443        state.merge = Some(MergeOutcome {
9444            mode: MergeMode::Local,
9445            ok: false,
9446            detail: "already concluded".to_owned(),
9447            empty: false,
9448        });
9449
9450        let mut runner = Runner {
9451            state,
9452            roles: ResolvedRoles {
9453                implementers: Vec::new(),
9454                judges: Vec::new(),
9455                reviewers: Vec::new(),
9456                fixer: None,
9457                conductor: conductor(),
9458                implementer_roster: Vec::new(),
9459                judge_roster: Vec::new(),
9460                reviewer_roster: Vec::new(),
9461            },
9462            sem: Arc::new(Semaphore::new(1)),
9463            pause: Pause::new(),
9464            interrupt: Pause::new(),
9465        };
9466
9467        runner.merge().await.expect("merge");
9468
9469        assert_eq!(
9470            runner.state.status,
9471            RunStatus::Ready,
9472            "a concluded run's status must not change on reentry"
9473        );
9474        assert_eq!(
9475            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
9476            Some("already concluded"),
9477            "merge must not run again once the node already recorded an outcome"
9478        );
9479    }
9480
9481    /// `gate` leaves `state.gate_ran` false both before it has ever run and
9482    /// when its last attempt was resource-blocked (the shared build cache
9483    /// could not be acquired or confirmed fresh in time - see
9484    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
9485    /// `Vec` this also leaves behind used to read as "nothing failed" and let
9486    /// a run merge a tree the gate never actually checked - exactly the case
9487    /// a contended cache produces on every retry until it clears. `merge`
9488    /// must refuse until `gate` has actually recorded an attempt.
9489    #[tokio::test]
9490    async fn merge_refuses_a_gate_that_has_not_actually_run() {
9491        let tmp = tempfile::tempdir().expect("tempdir");
9492        let repo = tmp.path().join("repo");
9493        std::fs::create_dir_all(&repo).unwrap();
9494        init_repo(&repo);
9495
9496        let mut config = Config::default();
9497        config.merge.mode = MergeMode::Local;
9498
9499        let mut state = RunState::new(
9500            repo.clone(),
9501            "main".to_owned(),
9502            "deadbeef".to_owned(),
9503            "task".to_owned(),
9504            config,
9505        );
9506        state.candidates = vec![Candidate {
9507            index: 0,
9508            label: 'A',
9509            agent: "alpha".to_owned(),
9510            branch: "does-not-exist".to_owned(),
9511            worktree: repo.clone(),
9512            summary: String::new(),
9513            stat: String::new(),
9514            files: 0,
9515            commits: 0,
9516            empty: false,
9517            failed: None,
9518            verified_noop: None,
9519            duration_ms: 0,
9520            folded: false,
9521        }];
9522        state.tally = Some(Tally {
9523            first_choice: BTreeMap::from([('A', 1)]),
9524            borda: BTreeMap::new(),
9525            winner: 'A',
9526            rankings: 1,
9527            unanimous_initial: true,
9528            deliberated: false,
9529            changed_votes: 0,
9530            unanimous_final: true,
9531            tie_break: None,
9532            judges: 0,
9533            present: 0,
9534            quorum: 0,
9535            met_quorum: true,
9536            uncontested: Some("only candidate A produced a change".to_owned()),
9537        });
9538        state.reviews = vec![ReviewRound {
9539            round: 1,
9540            head: "deadbeef".to_owned(),
9541            verified_head: None,
9542            verified_at: None,
9543            reviews: Vec::new(),
9544            e2e: Vec::new(),
9545            fix: None,
9546            blocking: 0,
9547            answered: 0,
9548            expected: 0,
9549            clean: true,
9550            verify_retried: false,
9551            e2e_deferred: false,
9552            e2e_defer_reason: None,
9553            progressed: false,
9554            vote_split: false,
9555            reconsideration: Vec::new(),
9556            verdict: None,
9557        }];
9558        // The point: `gate` has not recorded anything yet.
9559        state.gate = Vec::new();
9560        state.gate_ran = false;
9561        state.status = RunStatus::Gating;
9562
9563        let mut runner = Runner {
9564            state,
9565            roles: ResolvedRoles {
9566                implementers: Vec::new(),
9567                judges: Vec::new(),
9568                reviewers: Vec::new(),
9569                fixer: None,
9570                conductor: conductor(),
9571                implementer_roster: Vec::new(),
9572                judge_roster: Vec::new(),
9573                reviewer_roster: Vec::new(),
9574            },
9575            sem: Arc::new(Semaphore::new(1)),
9576            pause: Pause::new(),
9577            interrupt: Pause::new(),
9578        };
9579
9580        runner.merge().await.expect("merge");
9581
9582        assert!(
9583            runner.state.merge.is_none(),
9584            "an empty gate must never be read as a passing one: {:?}",
9585            runner.state.merge
9586        );
9587    }
9588
9589    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
9590    /// commands configured and `merge.mode` is `none` (a review-only run).
9591    /// `gate` must still record a real attempt — zero commands, vacuously
9592    /// passed — rather than leaving `state.gate` empty in a way `merge`
9593    /// cannot tell apart from "never ran"; otherwise the run reaches
9594    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
9595    #[tokio::test]
9596    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
9597        let tmp = tempfile::tempdir().expect("tempdir");
9598        let repo = tmp.path().join("repo");
9599        std::fs::create_dir_all(&repo).unwrap();
9600        init_repo(&repo);
9601
9602        // Default config: `verify.gate` empty, `merge.mode` is `none`.
9603        let config = Config::default();
9604
9605        let mut state = RunState::new(
9606            repo.clone(),
9607            "main".to_owned(),
9608            "deadbeef".to_owned(),
9609            "task".to_owned(),
9610            config,
9611        );
9612        state.candidates = vec![Candidate {
9613            index: 0,
9614            label: 'A',
9615            agent: "alpha".to_owned(),
9616            branch: "does-not-exist".to_owned(),
9617            worktree: repo.clone(),
9618            summary: String::new(),
9619            stat: String::new(),
9620            files: 0,
9621            commits: 0,
9622            empty: false,
9623            failed: None,
9624            verified_noop: None,
9625            duration_ms: 0,
9626            folded: false,
9627        }];
9628        state.tally = Some(Tally {
9629            first_choice: BTreeMap::from([('A', 1)]),
9630            borda: BTreeMap::new(),
9631            winner: 'A',
9632            rankings: 1,
9633            unanimous_initial: true,
9634            deliberated: false,
9635            changed_votes: 0,
9636            unanimous_final: true,
9637            tie_break: None,
9638            judges: 0,
9639            present: 0,
9640            quorum: 0,
9641            met_quorum: true,
9642            uncontested: Some("only candidate A produced a change".to_owned()),
9643        });
9644        state.reviews = vec![ReviewRound {
9645            round: 1,
9646            head: "deadbeef".to_owned(),
9647            verified_head: None,
9648            verified_at: None,
9649            reviews: Vec::new(),
9650            e2e: Vec::new(),
9651            fix: None,
9652            blocking: 0,
9653            answered: 0,
9654            expected: 0,
9655            clean: true,
9656            verify_retried: false,
9657            e2e_deferred: false,
9658            e2e_defer_reason: None,
9659            progressed: false,
9660            vote_split: false,
9661            reconsideration: Vec::new(),
9662            verdict: None,
9663        }];
9664
9665        let mut runner = Runner {
9666            state,
9667            roles: ResolvedRoles {
9668                implementers: Vec::new(),
9669                judges: Vec::new(),
9670                reviewers: Vec::new(),
9671                fixer: None,
9672                conductor: conductor(),
9673                implementer_roster: Vec::new(),
9674                judge_roster: Vec::new(),
9675                reviewer_roster: Vec::new(),
9676            },
9677            sem: Arc::new(Semaphore::new(1)),
9678            pause: Pause::new(),
9679            interrupt: Pause::new(),
9680        };
9681
9682        runner.gate().await.expect("gate");
9683        assert!(
9684            runner.state.gate_ran,
9685            "zero configured commands is still a real attempt, not an unrun gate"
9686        );
9687        assert!(runner.state.gate.is_empty());
9688        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
9689        assert_ne!(
9690            runner.state.status,
9691            RunStatus::Blocked,
9692            "a gate with nothing to check must not read as failed"
9693        );
9694
9695        runner.merge().await.expect("merge");
9696        assert_eq!(
9697            runner.state.status,
9698            RunStatus::Ready,
9699            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
9700        );
9701    }
9702
9703    /// `Config::cache_dir` is derived from `verify.e2e` as well as
9704    /// `verify.gate` (so the e2e leg and the final gate never build against
9705    /// different directories). With zero `verify.gate` commands but a
9706    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
9707    /// that lease before discovering it had nothing to run - so a repo with
9708    /// no gate commands could come back `resource_blocked` (and therefore
9709    /// still `gate_ran == false`) on nothing but an unrelated run holding the
9710    /// cache, exactly the contention this run's own zero commands could
9711    /// never have touched. `gate` must recognise there is nothing to check
9712    /// before it ever asks for the lease.
9713    #[tokio::test]
9714    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
9715        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9716        let home = crate::run::home();
9717
9718        let tmp = tempfile::tempdir().expect("tempdir");
9719        let repo = tmp.path().join("repo");
9720        std::fs::create_dir_all(&repo).unwrap();
9721        init_repo(&repo);
9722        // Unique to this test, so holding its lease cannot collide with
9723        // another test sharing the same process-wide `home`.
9724        let cache_dir = tmp.path().join("target");
9725
9726        let mut config = Config::default();
9727        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
9728        // `verify.gate` stays empty (the default). Bounded so a regression
9729        // that does start waiting fails the test in seconds, not hangs it.
9730        config.graph.timeout_verify = Some(2);
9731
9732        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
9733        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
9734            .expect("no io error acquiring directly")
9735        {
9736            crate::cache::AcquireOutcome::Acquired(g) => g,
9737            crate::cache::AcquireOutcome::Busy(b) => {
9738                panic!("expected the direct acquire to win the lease first: {b:?}")
9739            }
9740        };
9741
9742        let mut state = RunState::new(
9743            repo.clone(),
9744            "main".to_owned(),
9745            "deadbeef".to_owned(),
9746            "task".to_owned(),
9747            config,
9748        );
9749        state.candidates = vec![Candidate {
9750            index: 0,
9751            label: 'A',
9752            agent: "alpha".to_owned(),
9753            branch: "does-not-exist".to_owned(),
9754            worktree: repo.clone(),
9755            summary: String::new(),
9756            stat: String::new(),
9757            files: 0,
9758            commits: 0,
9759            empty: false,
9760            failed: None,
9761            verified_noop: None,
9762            duration_ms: 0,
9763            folded: false,
9764        }];
9765        state.tally = Some(Tally {
9766            first_choice: BTreeMap::from([('A', 1)]),
9767            borda: BTreeMap::new(),
9768            winner: 'A',
9769            rankings: 1,
9770            unanimous_initial: true,
9771            deliberated: false,
9772            changed_votes: 0,
9773            unanimous_final: true,
9774            tie_break: None,
9775            judges: 0,
9776            present: 0,
9777            quorum: 0,
9778            met_quorum: true,
9779            uncontested: Some("only candidate A produced a change".to_owned()),
9780        });
9781        state.reviews = vec![ReviewRound {
9782            round: 1,
9783            head: "deadbeef".to_owned(),
9784            verified_head: None,
9785            verified_at: None,
9786            reviews: Vec::new(),
9787            e2e: Vec::new(),
9788            fix: None,
9789            blocking: 0,
9790            answered: 0,
9791            expected: 0,
9792            clean: true,
9793            verify_retried: false,
9794            e2e_deferred: false,
9795            e2e_defer_reason: None,
9796            progressed: false,
9797            vote_split: false,
9798            reconsideration: Vec::new(),
9799            verdict: None,
9800        }];
9801
9802        let mut runner = Runner {
9803            state,
9804            roles: ResolvedRoles {
9805                implementers: Vec::new(),
9806                judges: Vec::new(),
9807                reviewers: Vec::new(),
9808                fixer: None,
9809                conductor: conductor(),
9810                implementer_roster: Vec::new(),
9811                judge_roster: Vec::new(),
9812                reviewer_roster: Vec::new(),
9813            },
9814            sem: Arc::new(Semaphore::new(1)),
9815            pause: Pause::new(),
9816            interrupt: Pause::new(),
9817        };
9818
9819        let started = std::time::Instant::now();
9820        runner.gate().await.expect("gate");
9821        assert!(
9822            started.elapsed() < Duration::from_secs(1),
9823            "a gate with nothing to run must never wait on a lease it never needed"
9824        );
9825        assert!(
9826            runner.state.gate_ran,
9827            "zero commands is still a real, immediate attempt"
9828        );
9829        assert!(runner.state.gate.is_empty());
9830        assert_ne!(
9831            runner.state.status,
9832            RunStatus::Blocked,
9833            "must not read as resource-blocked on a lease it never asked for"
9834        );
9835    }
9836
9837    /// The addendum's second gap: a `verify.gate` command running for real
9838    /// wall-clock time had nothing at all to show for it in `active` before
9839    /// `run_commands` learned to record it — a run could sit in `Gating` for
9840    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
9841    /// what was actually happening. Proven with a genuinely still-running
9842    /// command, not just a before/after check on the final state: a poller
9843    /// task reads the same `run.json` `gate()` is writing, the same way the
9844    /// phone or `magi show` would, while the shell command is still blocked
9845    /// on its own release marker.
9846    #[tokio::test]
9847    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
9848        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9849
9850        let tmp = tempfile::tempdir().expect("tempdir");
9851        let repo = tmp.path().join("repo");
9852        std::fs::create_dir_all(&repo).unwrap();
9853        init_repo(&repo);
9854
9855        let mut config = Config::default();
9856        config.verify.gate = vec![
9857            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
9858             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
9859                .to_owned(),
9860        ];
9861
9862        let mut state = RunState::new(
9863            repo.clone(),
9864            "main".to_owned(),
9865            "deadbeef".to_owned(),
9866            "task".to_owned(),
9867            config,
9868        );
9869        let run_id = state.id.clone();
9870        state.candidates = vec![Candidate {
9871            index: 0,
9872            label: 'A',
9873            agent: "alpha".to_owned(),
9874            branch: "does-not-exist".to_owned(),
9875            worktree: repo.clone(),
9876            summary: String::new(),
9877            stat: String::new(),
9878            files: 0,
9879            commits: 0,
9880            empty: false,
9881            failed: None,
9882            verified_noop: None,
9883            duration_ms: 0,
9884            folded: false,
9885        }];
9886        state.tally = Some(Tally {
9887            first_choice: BTreeMap::from([('A', 1)]),
9888            borda: BTreeMap::new(),
9889            winner: 'A',
9890            rankings: 1,
9891            unanimous_initial: true,
9892            deliberated: false,
9893            changed_votes: 0,
9894            unanimous_final: true,
9895            tie_break: None,
9896            judges: 0,
9897            present: 0,
9898            quorum: 0,
9899            met_quorum: true,
9900            uncontested: Some("only candidate A produced a change".to_owned()),
9901        });
9902        state.reviews = vec![ReviewRound {
9903            round: 1,
9904            head: "deadbeef".to_owned(),
9905            verified_head: None,
9906            verified_at: None,
9907            reviews: Vec::new(),
9908            e2e: Vec::new(),
9909            fix: None,
9910            blocking: 0,
9911            answered: 0,
9912            expected: 0,
9913            clean: true,
9914            verify_retried: false,
9915            e2e_deferred: false,
9916            e2e_defer_reason: None,
9917            progressed: false,
9918            vote_split: false,
9919            reconsideration: Vec::new(),
9920            verdict: None,
9921        }];
9922
9923        let mut runner = Runner {
9924            state,
9925            roles: ResolvedRoles {
9926                implementers: Vec::new(),
9927                judges: Vec::new(),
9928                reviewers: Vec::new(),
9929                fixer: None,
9930                conductor: conductor(),
9931                implementer_roster: Vec::new(),
9932                judge_roster: Vec::new(),
9933                reviewer_roster: Vec::new(),
9934            },
9935            sem: Arc::new(Semaphore::new(1)),
9936            pause: Pause::new(),
9937            interrupt: Pause::new(),
9938        };
9939
9940        let started_marker = repo.join("started.marker");
9941        let release_marker = repo.join("release.marker");
9942        let poller = tokio::spawn(async move {
9943            // Bounded so a regression that never records the task entry
9944            // fails this test in seconds instead of hanging the suite —
9945            // the same shape `a_park_requested_while_a_seat_is_mid_call_
9946            // does_not_cut_it_short` uses for the same reason.
9947            for _ in 0..100 {
9948                if started_marker.exists()
9949                    && let Ok(s) = crate::run::RunState::load(&run_id)
9950                    && let Some(a) = s.active.get("gate")
9951                {
9952                    std::fs::write(&release_marker, b"go").expect("release marker");
9953                    return Some(a.clone());
9954                }
9955                tokio::time::sleep(Duration::from_millis(50)).await;
9956            }
9957            None
9958        });
9959
9960        runner.gate().await.expect("gate");
9961        let captured = poller.await.expect("poller task");
9962        let captured = captured.expect(
9963            "the poller never saw a `gate` task entry in run.json while the command was \
9964             still blocked on its own release marker",
9965        );
9966
9967        assert_eq!(captured.task.as_deref(), Some("gate"));
9968        assert_eq!(captured.node, "gate");
9969        assert_eq!(captured.index, Some(1));
9970        assert_eq!(captured.total, Some(1));
9971        assert!(
9972            captured
9973                .command
9974                .as_deref()
9975                .is_some_and(|c| c.contains("started.marker")),
9976            "{captured:?}"
9977        );
9978
9979        assert!(
9980            runner.state.active.is_empty(),
9981            "the entry must be cleared once the command actually finished: {:?}",
9982            runner.state.active
9983        );
9984        assert!(runner.state.gate_ran);
9985        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
9986    }
9987
9988    /// The hand-off over a blocking finding a reviewer rejected on leaves a
9989    /// record for `land`; one with only a Minor, or no reject, leaves none.
9990    #[tokio::test]
9991    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
9992        use crate::verdict::{Finding, ReviewVote, Severity};
9993        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
9994        let tmp = tempfile::tempdir().expect("tempdir");
9995        let repo = tmp.path().join("repo");
9996        std::fs::create_dir_all(&repo).unwrap();
9997        init_repo(&repo);
9998
9999        for (severity, vote, expect) in [
10000            (Severity::Major, ReviewVote::Reject, true),
10001            (Severity::Minor, ReviewVote::Reject, false),
10002            (Severity::Major, ReviewVote::Approve, false),
10003        ] {
10004            let mut round = review_round(false, 1, 1, 1, false, true);
10005            round.reviews = vec![ReviewRecord {
10006                reviewer: 1,
10007                agent: "alpha".to_owned(),
10008                summary: String::new(),
10009                findings: vec![Finding {
10010                    id: "R1-1-1".to_owned(),
10011                    severity,
10012                    file: None,
10013                    line: None,
10014                    title: "t".to_owned(),
10015                    detail: String::new(),
10016                }],
10017                vote: Some(vote),
10018                failed: None,
10019                duration_ms: 0,
10020                attempts: 0,
10021            }];
10022            let mut state = RunState::new(
10023                repo.clone(),
10024                "main".to_owned(),
10025                "deadbeef".to_owned(),
10026                "task".to_owned(),
10027                Config::default(),
10028            );
10029            state.reviews = vec![round];
10030            let mut runner = Runner {
10031                state,
10032                roles: ResolvedRoles {
10033                    implementers: Vec::new(),
10034                    judges: Vec::new(),
10035                    reviewers: Vec::new(),
10036                    fixer: None,
10037                    conductor: conductor(),
10038                    implementer_roster: Vec::new(),
10039                    judge_roster: Vec::new(),
10040                    reviewer_roster: Vec::new(),
10041                },
10042                sem: Arc::new(Semaphore::new(1)),
10043                pause: Pause::new(),
10044                interrupt: Pause::new(),
10045            };
10046            let shell = runner.state.config.shell();
10047            runner
10048                .stop_reviewing("round budget spent", &shell, &repo)
10049                .await
10050                .expect("stop_reviewing");
10051            assert_eq!(runner.state.status, RunStatus::Gating);
10052            assert_eq!(
10053                runner.state.contested_handoff.is_some(),
10054                expect,
10055                "{severity:?} + {vote:?}"
10056            );
10057        }
10058    }
10059
10060    /// The shape the incident this whole fix responds to actually had: the
10061    /// round budget spent, the last round's own e2e blocked on the shared
10062    /// build cache (held here by a live pid — this test process — exactly
10063    /// `cache`'s own unit tests' pattern for "another owner, still alive"
10064    /// without forking a process). `stop_reviewing` must retry it — not
10065    /// silently leave the round looking untouched (the catch-up-only half of
10066    /// the bug), and not read the contention as a red `e2e` and block the
10067    /// run on it (the other half). Called directly, the same way
10068    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
10069    /// above exercises `gate`, so this never needs a real cargo build to
10070    /// reach: the lease is never released, so `with_cache_lease` never gets
10071    /// past acquiring it into anything that would need a real workspace.
10072    #[tokio::test]
10073    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
10074        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
10075        let home = crate::run::home();
10076
10077        let tmp = tempfile::tempdir().expect("tempdir");
10078        let repo = tmp.path().join("repo");
10079        std::fs::create_dir_all(&repo).unwrap();
10080        init_repo(&repo);
10081        let head = crate::git::rev_parse(&repo, "HEAD")
10082            .await
10083            .expect("rev-parse");
10084        // Unique to this test, so holding its lease cannot collide with
10085        // another test sharing the same process-wide `home`.
10086        let cache_dir = tmp.path().join("target");
10087
10088        let mut config = Config::default();
10089        config.verify.e2e = vec![format!(
10090            "CARGO_TARGET_DIR='{}' test -f README.md",
10091            cache_dir.display()
10092        )];
10093        config.graph.review_rounds = 1;
10094        // Bounded so a regression that does start waiting fails the test in
10095        // seconds, not hangs it.
10096        config.graph.timeout_verify = Some(2);
10097
10098        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10099        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10100            .expect("no io error acquiring directly")
10101        {
10102            crate::cache::AcquireOutcome::Acquired(g) => g,
10103            crate::cache::AcquireOutcome::Busy(b) => {
10104                panic!("expected the direct acquire to win the lease first: {b:?}")
10105            }
10106        };
10107
10108        let mut state = RunState::new(
10109            repo.clone(),
10110            "main".to_owned(),
10111            head.clone(),
10112            "task".to_owned(),
10113            config,
10114        );
10115        state.candidates = vec![Candidate {
10116            index: 0,
10117            label: 'A',
10118            agent: "alpha".to_owned(),
10119            branch: "does-not-exist".to_owned(),
10120            worktree: repo.clone(),
10121            summary: String::new(),
10122            stat: String::new(),
10123            files: 0,
10124            commits: 0,
10125            empty: false,
10126            failed: None,
10127            verified_noop: None,
10128            duration_ms: 0,
10129            folded: false,
10130        }];
10131        state.tally = Some(Tally {
10132            first_choice: BTreeMap::from([('A', 1)]),
10133            borda: BTreeMap::new(),
10134            winner: 'A',
10135            rankings: 1,
10136            unanimous_initial: true,
10137            deliberated: false,
10138            changed_votes: 0,
10139            unanimous_final: true,
10140            tie_break: None,
10141            judges: 0,
10142            present: 0,
10143            quorum: 0,
10144            met_quorum: true,
10145            uncontested: Some("only candidate A produced a change".to_owned()),
10146        });
10147        // The round budget's last round, deferred: `needs_catchup_run`'s
10148        // other trigger. `stop_reviewing`'s retry machinery must treat this
10149        // exactly like a resource-blocked attempt once it actually runs.
10150        state.reviews = vec![ReviewRound {
10151            round: 1,
10152            head: head.clone(),
10153            verified_head: None,
10154            verified_at: None,
10155            reviews: Vec::new(),
10156            e2e: Vec::new(),
10157            fix: None,
10158            blocking: 1,
10159            answered: 1,
10160            expected: 1,
10161            clean: false,
10162            verify_retried: false,
10163            e2e_deferred: true,
10164            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
10165            progressed: false,
10166            vote_split: false,
10167            reconsideration: Vec::new(),
10168            verdict: None,
10169        }];
10170
10171        let mut runner = Runner {
10172            state,
10173            roles: ResolvedRoles {
10174                implementers: Vec::new(),
10175                judges: Vec::new(),
10176                reviewers: Vec::new(),
10177                fixer: None,
10178                conductor: conductor(),
10179                implementer_roster: Vec::new(),
10180                judge_roster: Vec::new(),
10181                reviewer_roster: Vec::new(),
10182            },
10183            sem: Arc::new(Semaphore::new(1)),
10184            pause: Pause::new(),
10185            interrupt: Pause::new(),
10186        };
10187
10188        let shell = runner.state.config.shell();
10189        runner
10190            .stop_reviewing("round budget spent", &shell, &repo)
10191            .await
10192            .expect("stop_reviewing");
10193
10194        let last = runner.state.reviews.last().expect("round record");
10195        assert_eq!(
10196            last.e2e_status(),
10197            E2eStatus::ResourceBlocked,
10198            "the shared cache is still held; the attempt must read as blocked, not deferred or \
10199             failed: {last:?}"
10200        );
10201        assert_eq!(
10202            last.verified_head.as_deref(),
10203            Some(head.as_str()),
10204            "which commit this attempt targeted is known even though nothing finished checking \
10205             it"
10206        );
10207        let first_attempt_at = last
10208            .verified_at
10209            .expect("when this attempt ran is known too");
10210        assert_ne!(
10211            runner.state.status,
10212            RunStatus::Blocked,
10213            "contention is evidence about the machine, not the patch — it must not settle the \
10214             run as blocked: {:?}",
10215            runner.state.status
10216        );
10217        assert!(
10218            !runner
10219                .state
10220                .events
10221                .iter()
10222                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
10223            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
10224            runner.state.events
10225        );
10226
10227        // The cache is still held: a later reentry must retry the same
10228        // round's verification again — not leave it looking exactly as
10229        // untouched as the first blocked attempt, which is indistinguishable
10230        // from never having tried again at all.
10231        runner
10232            .stop_reviewing("round budget spent", &shell, &repo)
10233            .await
10234            .expect("stop_reviewing retry");
10235        assert_eq!(
10236            runner.state.reviews.len(),
10237            1,
10238            "no new round was started: {:?}",
10239            runner.state.reviews
10240        );
10241        let last = runner.state.reviews.last().expect("round record");
10242        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
10243        assert!(
10244            last.verified_at.expect("still known") > first_attempt_at,
10245            "a second reentry must be a fresh attempt, not a stale copy of the first"
10246        );
10247        assert_ne!(runner.state.status, RunStatus::Blocked);
10248
10249        held.release();
10250    }
10251
10252    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
10253    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
10254    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
10255    /// own top-of-function fast path (`review_conclusion`) correctly reads
10256    /// this shape as `None` rather than guessing `Blocked`, and the loop's
10257    /// own `for` range is empty once the round budget is spent, so
10258    /// `review_loop` must retry the check itself rather than silently doing
10259    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
10260    /// above exercises directly, but through `review_loop`'s own entry point
10261    /// this time, proving the wiring between the two rather than just the
10262    /// retry logic in isolation.
10263    #[tokio::test]
10264    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
10265        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
10266        let home = crate::run::home();
10267
10268        let tmp = tempfile::tempdir().expect("tempdir");
10269        let repo = tmp.path().join("repo");
10270        std::fs::create_dir_all(&repo).unwrap();
10271        init_repo(&repo);
10272        let head = crate::git::rev_parse(&repo, "HEAD")
10273            .await
10274            .expect("rev-parse");
10275        let cache_dir = tmp.path().join("target");
10276
10277        let mut config = Config::default();
10278        config.verify.e2e = vec![format!(
10279            "CARGO_TARGET_DIR='{}' test -f README.md",
10280            cache_dir.display()
10281        )];
10282        config.graph.review_rounds = 1;
10283        config.graph.timeout_verify = Some(2);
10284
10285        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
10286        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
10287            .expect("no io error acquiring directly")
10288        {
10289            crate::cache::AcquireOutcome::Acquired(g) => g,
10290            crate::cache::AcquireOutcome::Busy(b) => {
10291                panic!("expected the direct acquire to win the lease first: {b:?}")
10292            }
10293        };
10294
10295        let mut state = RunState::new(
10296            repo.clone(),
10297            "main".to_owned(),
10298            head.clone(),
10299            "task".to_owned(),
10300            config,
10301        );
10302        state.candidates = vec![Candidate {
10303            index: 0,
10304            label: 'A',
10305            agent: "alpha".to_owned(),
10306            branch: "does-not-exist".to_owned(),
10307            worktree: repo.clone(),
10308            summary: String::new(),
10309            stat: String::new(),
10310            files: 0,
10311            commits: 0,
10312            empty: false,
10313            failed: None,
10314            verified_noop: None,
10315            duration_ms: 0,
10316            folded: false,
10317        }];
10318        state.tally = Some(Tally {
10319            first_choice: BTreeMap::from([('A', 1)]),
10320            borda: BTreeMap::new(),
10321            winner: 'A',
10322            rankings: 1,
10323            unanimous_initial: true,
10324            deliberated: false,
10325            changed_votes: 0,
10326            unanimous_final: true,
10327            tie_break: None,
10328            judges: 0,
10329            present: 0,
10330            quorum: 0,
10331            met_quorum: true,
10332            uncontested: Some("only candidate A produced a change".to_owned()),
10333        });
10334        // The exact shape a prior process's `stop_reviewing` would have left
10335        // on disk: the round budget's last round, a real attempt already
10336        // made and already resource-blocked.
10337        state.reviews = vec![ReviewRound {
10338            round: 1,
10339            head: head.clone(),
10340            verified_head: Some(head.clone()),
10341            verified_at: Some(jiff::Timestamp::now()),
10342            reviews: Vec::new(),
10343            e2e: vec![CommandOutcome {
10344                command: format!(
10345                    "CARGO_TARGET_DIR='{}' test -f README.md",
10346                    cache_dir.display()
10347                ),
10348                code: None,
10349                output_tail: "waiting for the shared build cache".to_owned(),
10350                duration_ms: 0,
10351                resource_blocked: true,
10352            }],
10353            fix: None,
10354            blocking: 1,
10355            answered: 1,
10356            expected: 1,
10357            clean: false,
10358            verify_retried: false,
10359            e2e_deferred: false,
10360            e2e_defer_reason: None,
10361            progressed: false,
10362            vote_split: false,
10363            reconsideration: Vec::new(),
10364            verdict: None,
10365        }];
10366
10367        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
10368        let mut runner = Runner {
10369            state,
10370            roles: ResolvedRoles {
10371                implementers: Vec::new(),
10372                judges: Vec::new(),
10373                reviewers: Vec::new(),
10374                fixer: None,
10375                conductor: conductor(),
10376                implementer_roster: Vec::new(),
10377                judge_roster: Vec::new(),
10378                reviewer_roster: Vec::new(),
10379            },
10380            sem: Arc::new(Semaphore::new(1)),
10381            pause: Pause::new(),
10382            interrupt: Pause::new(),
10383        };
10384
10385        // The lease is still held throughout, so this reentry's own retry is
10386        // also contended — proving `review_loop` actually tried again (not
10387        // that it happened to succeed) is what the timestamp comparison
10388        // below is for.
10389        runner.review_loop().await.expect("review_loop");
10390
10391        assert_eq!(
10392            runner.state.reviews.len(),
10393            1,
10394            "no new round was started on top of the unresolved one: {:?}",
10395            runner.state.reviews
10396        );
10397        let last = &runner.state.reviews[0];
10398        assert_eq!(
10399            last.e2e_status(),
10400            E2eStatus::ResourceBlocked,
10401            "still contended: {last:?}"
10402        );
10403        assert!(
10404            last.verified_at.expect("still known") > first_attempt_at,
10405            "review_loop must have actually retried the check, not left it exactly as found"
10406        );
10407        assert_ne!(
10408            runner.state.status,
10409            RunStatus::Blocked,
10410            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
10411            runner.state.status
10412        );
10413
10414        held.release();
10415    }
10416
10417    #[tokio::test]
10418    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
10419        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
10420        let tmp = tempfile::tempdir().expect("tempdir");
10421        let repo = tmp.path().join("repo");
10422        std::fs::create_dir_all(&repo).unwrap();
10423        init_repo(&repo);
10424
10425        let mut config = Config::default();
10426        config.merge.mode = MergeMode::Pr;
10427        config.graph.land = true;
10428        config.graph.land_approval = false;
10429
10430        let mut state = RunState::new(
10431            repo.clone(),
10432            "main".to_owned(),
10433            "deadbeef".to_owned(),
10434            "task".to_owned(),
10435            config,
10436        );
10437        state.candidates = vec![Candidate {
10438            index: 0,
10439            label: 'A',
10440            agent: "alpha".to_owned(),
10441            branch: "does-not-exist".to_owned(),
10442            worktree: repo.clone(),
10443            summary: String::new(),
10444            stat: String::new(),
10445            files: 0,
10446            commits: 0,
10447            empty: false,
10448            failed: None,
10449            verified_noop: None,
10450            duration_ms: 0,
10451            folded: false,
10452        }];
10453        state.tally = Some(Tally {
10454            first_choice: BTreeMap::from([('A', 1)]),
10455            borda: BTreeMap::new(),
10456            winner: 'A',
10457            rankings: 1,
10458            unanimous_initial: true,
10459            deliberated: false,
10460            changed_votes: 0,
10461            unanimous_final: true,
10462            tie_break: None,
10463            judges: 0,
10464            present: 0,
10465            quorum: 0,
10466            met_quorum: true,
10467            uncontested: Some("only candidate A produced a change".to_owned()),
10468        });
10469        state.reviews = vec![ReviewRound {
10470            round: 1,
10471            head: "deadbeef".to_owned(),
10472            verified_head: None,
10473            verified_at: None,
10474            reviews: Vec::new(),
10475            e2e: Vec::new(),
10476            fix: None,
10477            blocking: 0,
10478            answered: 0,
10479            expected: 0,
10480            clean: true,
10481            verify_retried: false,
10482            e2e_deferred: false,
10483            e2e_defer_reason: None,
10484            progressed: false,
10485            vote_split: false,
10486            reconsideration: Vec::new(),
10487            verdict: None,
10488        }];
10489        state.gate = vec![CommandOutcome {
10490            command: "test".to_owned(),
10491            code: Some(0),
10492            output_tail: String::new(),
10493            duration_ms: 0,
10494            resource_blocked: false,
10495        }];
10496        state.gate_ran = true;
10497        // A first pass through `merge` already pushed and opened this pull
10498        // request; `status` is `Landing` because a previous call into `land`
10499        // parked or was interrupted before it reached a terminal outcome.
10500        state.status = RunStatus::Landing;
10501        state.merge = Some(MergeOutcome {
10502            mode: MergeMode::Pr,
10503            ok: true,
10504            detail: "https://example.invalid/x/y/pull/1".to_owned(),
10505            empty: false,
10506        });
10507
10508        // The Landing-resume shortcut calls `run_land` directly rather than
10509        // through `merge`, which is exactly the call site that used to skip
10510        // `settle_questions` - see the fixture below.
10511        ask_test_home();
10512        let store = ask::Questions::open();
10513        let q = ask_open_question(&store, &state.id);
10514
10515        let mut runner = Runner {
10516            state,
10517            roles: ResolvedRoles {
10518                implementers: Vec::new(),
10519                judges: Vec::new(),
10520                reviewers: Vec::new(),
10521                fixer: None,
10522                conductor: conductor(),
10523                implementer_roster: Vec::new(),
10524                judge_roster: Vec::new(),
10525                reviewer_roster: Vec::new(),
10526            },
10527            sem: Arc::new(Semaphore::new(1)),
10528            pause: Pause::new(),
10529            interrupt: Pause::new(),
10530        };
10531
10532        // `execute`, not `merge` directly: the Landing-resume shortcut lives
10533        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
10534        // exactly because `review_loop` would otherwise clobber the marker
10535        // first.
10536        runner.execute().await.expect("execute");
10537
10538        assert_eq!(
10539            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
10540            Some("https://example.invalid/x/y/pull/1"),
10541            "reentry must not push again or open a second pull request over the \
10542             one `land` is already watching"
10543        );
10544        assert_ne!(
10545            runner.state.status,
10546            RunStatus::Landing,
10547            "land could not actually reach the fake pull request, so it must \
10548             have given up rather than left the run silently parked forever"
10549        );
10550        // `land` could not reach the fake pull request, so it gave up into
10551        // `Blocked` - still resumable, so the question must not have been
10552        // swept just because this branch now also calls `settle_questions`.
10553        assert_eq!(runner.state.status, RunStatus::Blocked);
10554        assert!(
10555            store.get(&q.id).unwrap().status.open(),
10556            "Blocked is still alive; settle_questions must have been a no-op here"
10557        );
10558    }
10559
10560    fn state_with_round(round: ReviewRound) -> RunState {
10561        let mut s = RunState::new(
10562            PathBuf::from("/repo"),
10563            "main".to_owned(),
10564            "abc1234".to_owned(),
10565            "add retries".to_owned(),
10566            Config::default(),
10567        );
10568        s.reviews = vec![round];
10569        s
10570    }
10571
10572    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
10573        crate::verdict::Finding {
10574            id: id.to_owned(),
10575            severity,
10576            file: None,
10577            line: None,
10578            title: title.to_owned(),
10579            detail: String::new(),
10580        }
10581    }
10582
10583    #[test]
10584    fn pr_body_names_open_findings_and_declined_ones() {
10585        let round = ReviewRound {
10586            round: 2,
10587            head: "deadbee".to_owned(),
10588            verified_head: None,
10589            verified_at: None,
10590            reviews: vec![ReviewRecord {
10591                attempts: 0,
10592                reviewer: 1,
10593                agent: "alpha".to_owned(),
10594                summary: String::new(),
10595                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
10596                vote: None,
10597                failed: None,
10598                duration_ms: 0,
10599            }],
10600            e2e: vec![CommandOutcome {
10601                command: "cargo test".to_owned(),
10602                code: Some(0),
10603                output_tail: String::new(),
10604                duration_ms: 0,
10605                resource_blocked: false,
10606            }],
10607            verify_retried: false,
10608            e2e_deferred: false,
10609            e2e_defer_reason: None,
10610            fix: Some(FixRecord {
10611                agent: "alpha".to_owned(),
10612                addressed: Vec::new(),
10613                rejected: vec![crate::verdict::Rejection {
10614                    id: "R1-1-1".to_owned(),
10615                    why: "not reachable from any caller".to_owned(),
10616                }],
10617                notes: String::new(),
10618                committed: true,
10619                failed: None,
10620                duration_ms: 0,
10621                continuation: None,
10622            }),
10623            blocking: 0,
10624            answered: 1,
10625            expected: 1,
10626            clean: false,
10627            progressed: true,
10628            vote_split: false,
10629            reconsideration: Vec::new(),
10630            verdict: None,
10631        };
10632        let state = state_with_round(round);
10633        let body = pr_message(&state, 'A').body;
10634
10635        assert!(body.contains("add retries"), "the task must still be there");
10636        assert!(body.contains("R2-1-1"), "{body}");
10637        assert!(body.contains("unused import"), "{body}");
10638        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
10639        assert!(
10640            body.contains("not reachable from any caller"),
10641            "the reason it was declined: {body}"
10642        );
10643    }
10644
10645    #[test]
10646    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
10647        let round = ReviewRound {
10648            round: 1,
10649            head: "deadbee".to_owned(),
10650            verified_head: None,
10651            verified_at: None,
10652            reviews: vec![ReviewRecord {
10653                attempts: 0,
10654                reviewer: 1,
10655                agent: "alpha".to_owned(),
10656                summary: String::new(),
10657                findings: Vec::new(),
10658                vote: None,
10659                failed: None,
10660                duration_ms: 0,
10661            }],
10662            e2e: Vec::new(),
10663            verify_retried: false,
10664            e2e_deferred: false,
10665            e2e_defer_reason: None,
10666            fix: None,
10667            blocking: 0,
10668            answered: 1,
10669            expected: 1,
10670            clean: true,
10671            progressed: false,
10672            vote_split: false,
10673            reconsideration: Vec::new(),
10674            verdict: None,
10675        };
10676        let state = state_with_round(round);
10677        let body = pr_message(&state, 'A').body;
10678        assert!(!body.contains("Open review findings"), "{body}");
10679        assert!(!body.contains("Declined"), "{body}");
10680    }
10681
10682    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
10683        let mut state = RunState::new(
10684            PathBuf::from("/repo"),
10685            "main".to_owned(),
10686            "abc1234".to_owned(),
10687            instruction.to_owned(),
10688            Config::default(),
10689        );
10690        state.candidates.push(Candidate {
10691            index: 0,
10692            label: 'A',
10693            agent: "alpha".to_owned(),
10694            branch: "magi/x/A".to_owned(),
10695            worktree: PathBuf::from("/wt"),
10696            summary: summary.to_owned(),
10697            stat: String::new(),
10698            files: 1,
10699            commits: 1,
10700            empty: false,
10701            failed: None,
10702            verified_noop: None,
10703            folded: false,
10704            duration_ms: 0,
10705        });
10706        state
10707    }
10708
10709    fn review_state(subjects: &[&str]) -> RunState {
10710        let mut state = state_with_summary(
10711            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
10712            "",
10713        );
10714        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
10715        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
10716        state
10717    }
10718
10719    #[test]
10720    fn pr_message_review_single_commit_uses_its_subject() {
10721        let state = review_state(&["feat(nats): per-role user"]);
10722        let m = pr_message(&state, 'A');
10723        assert_eq!(m.title, "feat(nats): per-role user");
10724        assert!(!m.body.contains("Review the work already"), "{}", m.body);
10725        assert!(m.body.contains("## Commits under review"), "{}", m.body);
10726    }
10727
10728    #[test]
10729    fn pr_message_review_multi_commit_takes_the_oldest() {
10730        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
10731        let m = pr_message(&state, 'A');
10732        assert_eq!(m.title, "feat: the change");
10733        for s in ["feat: the change", "fix: typo", "fix: again"] {
10734            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
10735        }
10736    }
10737
10738    #[test]
10739    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
10740        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
10741            let mut state = review_state(&[first, "fix: later fixup"]);
10742            state.candidates[0].branch = "機能/ログイン".to_owned();
10743            let m = pr_message(&state, 'A');
10744            assert!(
10745                m.title.starts_with("chore: land candidate A of run"),
10746                "{}",
10747                m.title
10748            );
10749        }
10750    }
10751
10752    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
10753        BranchFacts {
10754            commits: commits
10755                .iter()
10756                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
10757                .collect(),
10758            stat: stat.to_owned(),
10759        }
10760    }
10761
10762    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
10763        let mut r = review_round(true, 0, 1, 1, true, true);
10764        r.round = round;
10765        r.fix = notes.map(|n| FixRecord {
10766            agent: "fixer".to_owned(),
10767            addressed: Vec::new(),
10768            rejected: Vec::new(),
10769            notes: n.to_owned(),
10770            committed: true,
10771            failed: None,
10772            duration_ms: 0,
10773            continuation: None,
10774        });
10775        r
10776    }
10777
10778    #[test]
10779    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
10780        let state = review_state(&["ignored"]);
10781        let f = facts(
10782            &[
10783                (
10784                    "fix(login): resolve PATH on macOS",
10785                    "Login shells skip rc files.",
10786                ),
10787                ("fix: address review", ""),
10788            ],
10789            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
10790        );
10791        let m = pr_message_with(&state, 'A', Some(&f));
10792        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
10793        assert!(
10794            m.body
10795                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
10796            "{}",
10797            m.body
10798        );
10799        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
10800        assert!(m.body.contains("## Diff stat"), "{}", m.body);
10801        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
10802        for banned in [
10803            "Review the work already",
10804            "no task statement",
10805            "Original task",
10806        ] {
10807            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
10808        }
10809        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
10810    }
10811
10812    #[test]
10813    fn pr_message_review_truncates_a_huge_first_commit_body() {
10814        let state = review_state(&["ignored"]);
10815        let f = facts(
10816            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
10817            "s",
10818        );
10819        let m = pr_message_with(&state, 'A', Some(&f));
10820        assert!(m.body.len() < 30_000, "{}", m.body.len());
10821        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
10822        assert!(
10823            m.body.contains("- ... 1 more commit(s)"),
10824            "{}",
10825            m.body.len()
10826        );
10827        assert!(m.body.ends_with("magi:candidate-a\n"));
10828    }
10829
10830    #[test]
10831    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
10832        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
10833        assert_eq!(m.title, "feat: x");
10834        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
10835        assert!(!m.body.contains("Diff stat"), "{}", m.body);
10836        assert!(!m.body.contains("no task statement"), "{}", m.body);
10837    }
10838
10839    #[test]
10840    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
10841        let mut state = review_state(&["日本語の件名"]);
10842        state.candidates[0].branch = "fix/macos-login-path".to_owned();
10843        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
10844        state.candidates[0].branch = "機能/ログイン".to_owned();
10845        assert!(
10846            pr_message(&state, 'A')
10847                .title
10848                .starts_with("chore: land candidate A")
10849        );
10850    }
10851
10852    #[test]
10853    fn pr_message_review_fixes_survive_a_clean_final_round() {
10854        let mut state = review_state(&["feat: x"]);
10855        state.reviews = vec![
10856            round_with_notes(1, Some("handled the PATH case")),
10857            round_with_notes(2, None),
10858        ];
10859        let body = pr_message(&state, 'A').body;
10860        assert!(
10861            body.contains("## Review fixes\n\nhandled the PATH case\n"),
10862            "{body}"
10863        );
10864        assert!(!body.contains("### Round"), "{body}");
10865
10866        state.reviews = vec![
10867            round_with_notes(1, Some("first fix")),
10868            round_with_notes(2, Some("")),
10869            round_with_notes(3, Some("second fix")),
10870            round_with_notes(4, None),
10871        ];
10872        let body = pr_message(&state, 'A').body;
10873        assert!(body.contains("### Round 1\n\nfirst fix"), "{body}");
10874        assert!(body.contains("### Round 3\n\nsecond fix"), "{body}");
10875        assert!(!body.contains("### Round 2"), "{body}");
10876    }
10877
10878    #[test]
10879    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
10880        let state = state_with_summary(
10881            "add retries to the client",
10882            "TITLE: feat: retries\n\nDid it.",
10883        );
10884        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
10885        assert_eq!(m.title, "feat: retries");
10886        assert!(
10887            m.body.contains("<summary>Original task</summary>"),
10888            "{}",
10889            m.body
10890        );
10891        assert!(!m.body.contains("Commits under review"), "{}", m.body);
10892        assert!(
10893            m.body
10894                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
10895            "{}",
10896            m.body
10897        );
10898    }
10899
10900    #[test]
10901    fn pr_message_review_bounds_a_long_english_subject() {
10902        let long = format!("feat: {}", "word ".repeat(100));
10903        let m = pr_message(&review_state(&[&long]), 'A');
10904        assert!(m.title.starts_with("feat: word"), "{}", m.title);
10905        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
10906    }
10907
10908    #[test]
10909    fn pr_message_implementation_run_is_unchanged_by_review_support() {
10910        let state = state_with_summary("add retries\n\ndetails", "- did some things");
10911        let m = pr_message(&state, 'A');
10912        assert_eq!(m.title, "add retries");
10913        assert!(m.body.contains("<summary>Original task</summary>"));
10914        assert!(!m.body.contains("Commits under review"));
10915        assert_eq!(landing_subject_source(&state), state.instruction);
10916    }
10917
10918    #[test]
10919    fn review_run_squash_subject_is_the_change_not_the_prompt() {
10920        let state = review_state(&["feat: the change", "fix: typo"]);
10921        let source = landing_subject_source(&state);
10922        assert_eq!(land::merge_subject("", &source), "feat: the change");
10923        assert_eq!(
10924            land::merge_subject("magi: candidate A (uncommitted work)", &source),
10925            "feat: the change"
10926        );
10927        // An operator's rename still wins.
10928        assert_eq!(
10929            land::merge_subject("feat: renamed by hand", &source),
10930            "feat: renamed by hand"
10931        );
10932        let blank = review_state(&["日本語"]);
10933        assert!(
10934            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
10935        );
10936    }
10937
10938    #[test]
10939    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
10940        let state = review_state(&["feat: the change"]);
10941        let source = landing_subject_source(&state);
10942        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
10943        assert_eq!(
10944            land::merge_subject(landing_title(&state, old), &source),
10945            "feat: the change"
10946        );
10947        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
10948        let task = state_with_summary("add retries", "");
10949        assert_eq!(landing_title(&task, old), old);
10950    }
10951
10952    #[test]
10953    fn pr_message_describes_the_change_not_the_task() {
10954        let state = state_with_summary(
10955            "今回やってほしいこと: results projector を直す",
10956            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
10957        );
10958        let m = pr_message(&state, 'A');
10959        assert_eq!(m.title, "fix(web): batch the runs list reads");
10960        assert!(
10961            m.body.starts_with("## Summary\n\n- reads run.json once"),
10962            "{}",
10963            m.body
10964        );
10965        assert!(!m.body.contains("TITLE:"), "{}", m.body);
10966        let task_at = m.body.find("今回やってほしいこと").unwrap();
10967        let details_at = m.body.find("<details>").unwrap();
10968        assert!(
10969            details_at < task_at,
10970            "the task lives inside <details>: {}",
10971            m.body
10972        );
10973        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
10974        assert!(m.body.contains("magi:candidate-a"));
10975    }
10976
10977    #[test]
10978    fn pr_message_falls_back_to_the_task_without_a_title_line() {
10979        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
10980        let m = pr_message(&state, 'A');
10981        assert_eq!(m.title, "add retries");
10982        assert!(
10983            m.body.contains("## Summary\n\n- did some things"),
10984            "{}",
10985            m.body
10986        );
10987
10988        let none = RunState::new(
10989            PathBuf::from("/repo"),
10990            "main".to_owned(),
10991            "abc1234".to_owned(),
10992            "add retries".to_owned(),
10993            Config::default(),
10994        );
10995        let m = pr_message(&none, 'A');
10996        assert_eq!(m.title, "add retries");
10997        assert!(!m.body.contains("## Summary"), "{}", m.body);
10998    }
10999
11000    #[test]
11001    fn pr_message_refuses_the_candidate_commit_subject() {
11002        for bad in [
11003            "TITLE: magi: candidate A (uncommitted work)",
11004            "TITLE: chore: stuff (uncommitted work)",
11005            "TITLE:   ",
11006        ] {
11007            let state = state_with_summary("add retries", bad);
11008            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
11009        }
11010    }
11011
11012    #[test]
11013    fn pr_message_bounds_a_very_long_task_and_title() {
11014        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
11015        let state = state_with_summary(&long, "- nothing");
11016        let m = pr_message(&state, 'A');
11017        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11018        assert!(!m.title.contains('\n'));
11019
11020        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
11021        let m = pr_message(&state, 'A');
11022        assert!(m.title.starts_with("feat: "));
11023        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
11024        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
11025    }
11026
11027    #[test]
11028    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
11029        // What magi itself writes stays English under any configured language,
11030        // so a future localisation of these headings fails here. (The agents'
11031        // own text is held to English by the prompt only; magi cannot check it.)
11032        let mut state = state_with_summary(
11033            "add retries",
11034            "TITLE: fix(web): batch reads\n- reads run.json once",
11035        );
11036        state.config.graph.language = "ja".to_owned();
11037        let m = pr_message(&state, 'A');
11038        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
11039
11040        // The task is the operator's own text: it goes in untouched, and the
11041        // fallback title (no summary) may be in its language too.
11042        let task = "今回やってほしいこと: results projector を直す";
11043        let mut state = state_with_summary(task, "- no title line");
11044        state.config.graph.language = "ja".to_owned();
11045        let m = pr_message(&state, 'A');
11046        assert_eq!(
11047            m.title,
11048            format!("chore: land candidate A of run {}", state.id)
11049        );
11050        assert!(
11051            m.body.contains(&format!(
11052                "<summary>Original task</summary>\n\n{task}\n\n</details>"
11053            )),
11054            "{}",
11055            m.body
11056        );
11057    }
11058
11059    #[test]
11060    fn pr_message_scrubs_home_paths_and_addresses() {
11061        let state = state_with_summary(
11062            "fix it in /Users/someone/src/x",
11063            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
11064        );
11065        let m = pr_message(&state, 'A');
11066        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
11067            assert!(!m.body.contains(leak), "{}", m.body);
11068        }
11069        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
11070    }
11071
11072    #[test]
11073    fn pr_message_survives_a_task_that_closes_details() {
11074        let state = state_with_summary("a </details> b", "TITLE: fix: x");
11075        let m = pr_message(&state, 'A');
11076        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
11077    }
11078
11079    #[test]
11080    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
11081        let cmd = manual_merge_command(
11082            MergeStyle::Squash,
11083            Path::new("/repo"),
11084            "b",
11085            "fix: \"quoted\" $(x) `y`\n\nbody",
11086        );
11087        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
11088    }
11089
11090    #[test]
11091    fn manual_merge_command_matches_the_configured_style() {
11092        let repo = Path::new("/repo");
11093        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
11094
11095        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
11096        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
11097
11098        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
11099        assert_eq!(
11100            squash,
11101            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
11102             \"Merge magi run 0832 (candidate A)\""
11103        );
11104
11105        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
11106        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
11107    }
11108
11109    #[test]
11110    fn a_nudge_gets_a_quarter_of_the_budget() {
11111        // The judge and implement budgets magi ships with.
11112        assert_eq!(retry_budget(secs(1200), true), secs(300));
11113        assert_eq!(retry_budget(secs(3600), true), secs(900));
11114    }
11115
11116    #[test]
11117    fn a_resent_prompt_keeps_the_whole_budget() {
11118        // The seat kept no context, so the retry is the original job again and
11119        // shortening it would only guarantee a second failure.
11120        assert_eq!(retry_budget(secs(1200), false), secs(1200));
11121        assert_eq!(retry_budget(secs(60), false), secs(60));
11122    }
11123
11124    #[test]
11125    fn the_floor_never_exceeds_the_original_budget() {
11126        // A short configured timeout must not be *raised* by the floor: the
11127        // operator asked for a bound, and a retry may not outlast the attempt
11128        // it is retrying.
11129        assert_eq!(retry_budget(secs(60), true), secs(60));
11130        assert_eq!(retry_budget(secs(480), true), secs(120));
11131        assert_eq!(retry_budget(secs(0), true), secs(0));
11132    }
11133
11134    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
11135        agent::CommandEvidence {
11136            id: "item1".to_owned(),
11137            description: "cargo test".to_owned(),
11138            exit_code,
11139            result_summary: String::new(),
11140            source: "codex".to_owned(),
11141        }
11142    }
11143
11144    #[test]
11145    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
11146        // No evidence is not the same fact as unconfirmed evidence: a
11147        // backend with no adapter, or a reply that ran no commands at all,
11148        // must not be misread as carrying a dangling job.
11149        assert!(!has_unconfirmed_command(&[]));
11150    }
11151
11152    #[test]
11153    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
11154        // Deliberately not a check on the exit code's *value*: a fixer
11155        // legitimately runs something that fails mid-iteration before it
11156        // succeeds, and that must never by itself reopen a valid report.
11157        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
11158        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
11159        assert!(!has_unconfirmed_command(&[
11160            evidence(Some(0)),
11161            evidence(Some(101))
11162        ]));
11163    }
11164
11165    #[test]
11166    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
11167        assert!(has_unconfirmed_command(&[
11168            evidence(Some(0)),
11169            evidence(None)
11170        ]));
11171    }
11172
11173    #[test]
11174    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
11175        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11176        assert_eq!(
11177            verified_noop_claim(true, &[], text).as_deref(),
11178            Some("already fixed by b32cfc4, on main.")
11179        );
11180    }
11181
11182    #[test]
11183    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
11184        // A timeout or a bad exit code reads as the ordinary loss it is,
11185        // whatever the reply's own prose claims.
11186        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11187        assert!(verified_noop_claim(false, &[], text).is_none());
11188    }
11189
11190    #[test]
11191    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
11192        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
11193        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
11194        // A confirmed command alongside the marker is fine.
11195        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
11196    }
11197
11198    #[test]
11199    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
11200        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
11201    }
11202
11203    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
11204    /// pair, in order, labelled A, B, C, ...
11205    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
11206        runner.state.candidates = shape
11207            .iter()
11208            .enumerate()
11209            .map(|(i, &(empty, verified))| Candidate {
11210                index: i,
11211                label: (b'A' + i as u8) as char,
11212                agent: "sonnet".to_owned(),
11213                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
11214                worktree: PathBuf::from(format!("/wt/{i}")),
11215                summary: String::new(),
11216                stat: String::new(),
11217                files: 0,
11218                commits: 0,
11219                empty,
11220                failed: None,
11221                verified_noop: verified.map(str::to_owned),
11222                duration_ms: 0,
11223                folded: false,
11224            })
11225            .collect();
11226    }
11227
11228    #[test]
11229    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
11230        ask_test_home();
11231        let mut runner = runner_at(RunStatus::Implementing);
11232        set_candidates(
11233            &mut runner,
11234            &[
11235                (true, Some("already on main at b32cfc4")),
11236                (true, Some("same fix, see the existing test")),
11237            ],
11238        );
11239
11240        runner
11241            .after_implement()
11242            .expect("a verified no-op is not an error");
11243
11244        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
11245    }
11246
11247    #[test]
11248    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
11249        ask_test_home();
11250        let mut runner = runner_at(RunStatus::Implementing);
11251        // Candidate A declares a verified no-op; candidate B simply wrote
11252        // nothing and said nothing about why. One candidate's claim is not
11253        // the whole run's agreement.
11254        set_candidates(
11255            &mut runner,
11256            &[(true, Some("already on main at b32cfc4")), (true, None)],
11257        );
11258
11259        let err = runner
11260            .after_implement()
11261            .expect_err("an unverified empty candidate must still fail the run");
11262
11263        assert!(
11264            err.to_string().contains("no candidate produced a change"),
11265            "{err}"
11266        );
11267        assert_eq!(runner.state.status, RunStatus::Failed);
11268    }
11269
11270    #[test]
11271    fn after_implement_still_fails_an_ordinary_all_empty_run() {
11272        ask_test_home();
11273        let mut runner = runner_at(RunStatus::Implementing);
11274        set_candidates(&mut runner, &[(true, None), (true, None)]);
11275
11276        let err = runner
11277            .after_implement()
11278            .expect_err("no candidate declared anything; this is an ordinary failure");
11279
11280        assert!(
11281            err.to_string().contains("no candidate produced a change"),
11282            "{err}"
11283        );
11284        assert_eq!(runner.state.status, RunStatus::Failed);
11285    }
11286}