Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::run::{
45    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
46    DeliberationRound, DeliberationTurn, FixRecord, JobRecord, JobStatus, Judgement, MergeOutcome,
47    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, Tally,
48    VoteRecord, tail, write_artifact,
49};
50use crate::verdict::{
51    self, FinalVote, FixReport, Position, Proposal, Ranking, Review, ReviewRevote, ReviewVote,
52    Severity,
53};
54
55/// How much verification output is kept and fed back to the fixer.
56const OUTPUT_TAIL: usize = 8_000;
57
58/// Bytes of a failing command's output kept in an event, so the reason a run
59/// stopped is readable from the report without opening `run.json`.
60const EVENT_OUTPUT_TAIL: usize = 2_000;
61
62/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
63/// command's pid before releasing the build cache's lease.
64const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
65
66/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
67/// command's pid to actually exit before giving up and releasing anyway.
68///
69/// A timeout means the process was asked to die (`kill_on_drop`,
70/// `start_kill`), not that it already has — on Windows in particular that can
71/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
72/// the instant the command returns would let the very next acquirer (this
73/// run's own next round, another run's verification, the janitor's prune)
74/// start touching the same directory while it might still be writing to it,
75/// so this polls the actual pid — real confirmation, not a fixed guess —
76/// until it is gone or this ceiling is reached. It is still not full
77/// process-tree reaping: a grandchild the timed-out process spawned and that
78/// outlives it independently is invisible to a pid check, and continuing to
79/// observe and collect *that* stays a different piece of work with its own
80/// owner. Set generously because the common case returns early the moment
81/// the pid is confirmed gone, not because every timeout pays this in full.
82const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
83
84/// Consecutive review rounds with no tree progress (see
85/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
86/// instead of spending the rest of the round budget.
87///
88/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
89/// legitimately finds nothing left to change (its previous round's fix already
90/// covered it, and this round's reviewers re-raised only nits) looks the same
91/// as one that is spinning, for exactly one round. Two in a row is where the
92/// two stop being distinguishable, and a review round on this workload has
93/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
94/// third attempt at a tree that has not moved twice running is pure cost.
95/// This does not touch `review_rounds` itself, which stays the operator's
96/// call.
97pub(crate) const STAGNANT_LIMIT: usize = 2;
98
99/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
100/// a base that moved before giving up and leaving the run `Blocked` for a
101/// person.
102///
103/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
104/// that keeps moving faster than a run can catch it is not something more
105/// rebasing fixes, it is a person's call. Not the same *number as*
106/// `land_rounds` - this budget is spent before a pull request exists, land's
107/// after - but bounded for the identical reason, so it uses the same
108/// default. Counted across both call sites in [`Runner::finish_after_tally`]
109/// (once before review, once before the gate), because either one finding
110/// the base still moving is the same signal.
111const BASE_SYNC_ROUNDS: usize = 4;
112
113/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
114/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
115/// reply held no [`FixReport`].
116///
117/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
118/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
119/// "I'll pause here until the `cargo make check` background run reports
120/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
121/// No `FixReport` was ever collected from that seat, and the run moved on to
122/// the next review round regardless.
123///
124/// Bounded independently of `review_rounds` and `graph.retries`: this
125/// recovers one seat's missing report mid-round, not a new round of review or
126/// an ordinary parse retry, and must not itself become the unbounded wait the
127/// rest of this module exists to avoid.
128const MAX_FIX_CONTINUATIONS: usize = 2;
129
130/// One queued agent invocation.
131///
132/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
133/// CLI hung up on its own stream is asked again from the same job rather than
134/// rebuilt from scratch. See [`Runner::resume_undelivered`].
135#[derive(Clone)]
136struct SeatJob {
137    spec: AgentSpec,
138    seat: SeatState,
139    cwd: PathBuf,
140    prompt: String,
141    timeout: Duration,
142    allow_write: bool,
143    sessions: bool,
144    artifacts: PathBuf,
145    stem: String,
146}
147
148/// How the graph reads one agent invocation.
149///
150/// Quota is split out from an ordinary failure on purpose: a rate-limited call
151/// is known to fail again if retried now, so the retry loop must not spend an
152/// attempt on it. `Dropped` is split out for the opposite reason: unlike
153/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
154/// error JSON, never the agent's answer — a caller that matched only
155/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
156/// left to read that JSON as if it were usable output. `resume_undelivered`
157/// is the only caller that acts on it; everywhere else it is reported like an
158/// ordinary failure.
159enum AgentOutcome {
160    /// A usable output.
161    Ok(AgentOutput),
162    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
163    Quota(AgentOutput),
164    /// The CLI hung up on its own stream after billed work. See
165    /// [`agent::AgentOutput::work_undelivered`].
166    Dropped(AgentOutput),
167    /// Any other failure: a timeout, a bad exit code, an empty reply.
168    Failed(String),
169}
170
171/// A request to park the run at its next node boundary.
172///
173/// Cloning is how the request travels: the loop keeps one handle and hands a
174/// clone to each [`Runner`], and every clone points at the same flag. There
175/// is no channel because there is nothing to send - the only message is
176/// "park", it is idempotent, and a flag cannot be missed by a receiver that
177/// was not listening yet.
178///
179/// The boundary is what makes this cheap. Every node writes the run's state
180/// before the next one starts, and every node skips what is already recorded:
181/// `prep` returns early once candidates exist, `implement` asks only the seats
182/// with nothing on disk, `judge` returns early once judgements exist. So a
183/// parked run resumes into exactly the node it stopped before, and no agent
184/// work is thrown away. Killing the process mid-node, by contrast, loses
185/// whatever the seats in flight had not yet written - which for an implement
186/// wave is an hour of paid work.
187///
188/// A [`Runner`] watches two independent handles of this type - see
189/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
190/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
191/// clone covering the whole daemon's lifetime and is never asked to un-park,
192/// which is correct exactly because nothing is dispatched after it fires.
193/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
194/// that parks for an interrupted task must go on to run other tasks
195/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
196/// reusing the daemon-wide one.
197#[derive(Debug, Clone, Default)]
198pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<String>>>);
199
200impl Pause {
201    /// A pause nobody has asked for yet.
202    #[must_use]
203    pub fn new() -> Self {
204        Self::default()
205    }
206
207    /// Ask the run to park at its next node boundary. Idempotent.
208    pub fn park(&self) {
209        self.0.store(true, Ordering::SeqCst);
210    }
211
212    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
213    /// fold into the run's own `park` event - so an operator reading the run
214    /// later knows this was a deliberate interrupt rather than a shutdown or
215    /// a binary swap. The first reason recorded wins; a park already in
216    /// flight is not relabelled by a second, unrelated request.
217    pub fn park_because(&self, reason: impl Into<String>) {
218        let mut reason_guard = self
219            .1
220            .lock()
221            .unwrap_or_else(std::sync::PoisonError::into_inner);
222        if reason_guard.is_none() {
223            *reason_guard = Some(reason.into());
224        }
225        drop(reason_guard);
226        self.park();
227    }
228
229    /// Has a park been asked for?
230    #[must_use]
231    pub fn parked(&self) -> bool {
232        self.0.load(Ordering::SeqCst)
233    }
234
235    /// Why the park was asked for, when the caller used [`Pause::park_because`].
236    #[must_use]
237    pub fn reason(&self) -> Option<String> {
238        self.1
239            .lock()
240            .unwrap_or_else(std::sync::PoisonError::into_inner)
241            .clone()
242    }
243}
244
245/// Drives one run.
246pub struct Runner {
247    /// Run state; public so the CLI can report on it.
248    pub state: RunState,
249    roles: ResolvedRoles,
250    sem: Arc<Semaphore>,
251    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
252    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
253    /// this is never the same handle as `interrupt`.
254    pause: Pause,
255    /// Set when `magi serve`'s interrupt scheduler wants this specific run
256    /// parked at its next node boundary, to let a task marked
257    /// [`crate::queue::Task::interrupt`] run alone before this one carries
258    /// on. Unlike `pause`, a fresh, unshared handle per run - see
259    /// [`Runner::watch_interrupt`].
260    interrupt: Pause,
261}
262
263/// The commit a run branches from: the base branch as the remote has it.
264///
265/// Two failures this replaces. A run used to branch off `HEAD` and so refused
266/// to start on a dirty tree, which made `magi serve` decline every task for as
267/// long as the operator had work in progress - most of the time. Branching off
268/// the *local* base branch fixed that and introduced a worse one: `land` merges
269/// the winner on GitHub, nothing updates the local ref, and the next run
270/// branches off a base missing everything the previous runs landed. Two tasks
271/// in a row from a phone would have had the second silently re-implementing
272/// against stale code and opening a pull request that reverted the first.
273///
274/// Only refs move here - no checkout, no local branch, no merge - so it is safe
275/// with uncommitted work in the tree. A machine with no network still starts:
276/// the fetch may fail and the local tip is used with a warning, because
277/// refusing to run offline is a worse failure than running against a base the
278/// operator can see for themselves.
279///
280/// One function, called by both entry points. Two answers to "where does a run
281/// branch from" is the kind of drift nobody notices until a diff is wrong.
282async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
283    let tracking = format!("{remote}/{base_branch}");
284    let fetched = git::fetch(repo, remote, base_branch).await;
285    if let Ok(out) = &fetched
286        && out.ok()
287        && git::rev_exists(repo, &tracking).await
288    {
289        return git::rev_parse(repo, &tracking).await;
290    }
291    let why = match &fetched {
292        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
293        Ok(_) => format!("{remote} has no {base_branch}"),
294        Err(e) => e.to_string(),
295    };
296    tracing::warn!(
297        "could not read {tracking} ({why}); branching off the local \
298         {base_branch} instead, which may be behind"
299    );
300    git::rev_parse(repo, base_branch).await.with_context(|| {
301        format!(
302            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
303             branch that exists"
304        )
305    })
306}
307
308impl Runner {
309    /// Start a fresh run against `repo`.
310    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
311        let repo = git::toplevel(repo).await?;
312        let missing = agent::missing_programs(&config.agents);
313        if !missing.is_empty() {
314            bail!(
315                "these agent programs are not on PATH: {}. Fix the roster in \
316                 magi.toml or install them.",
317                missing.join(", ")
318            );
319        }
320        let base_branch = match config.merge.base.clone() {
321            Some(b) => b,
322            None => git::current_branch(&repo)
323                .await?
324                .context("HEAD is detached; set [merge] base in magi.toml")?,
325        };
326        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
327        // Still worth saying out loud. The operator's uncommitted work is not
328        // part of this run, and someone watching a candidate fail to use a
329        // change they just made deserves to know why.
330        if !git::is_clean(&repo).await? {
331            tracing::warn!(
332                "{} has uncommitted changes; they are not part of this run, \
333                 which branches off {base_branch} ({})",
334                repo.display(),
335                &base_commit[..base_commit.len().min(8)]
336            );
337        }
338        let roles = config.resolve_roles()?;
339        let max_parallel = config.graph.max_parallel.max(1);
340        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
341        state.event("start", format!("run {} created", state.id));
342        state.save()?;
343        Ok(Self {
344            state,
345            roles,
346            sem: Arc::new(Semaphore::new(max_parallel)),
347            pause: Pause::new(),
348            interrupt: Pause::new(),
349        })
350    }
351
352    /// Open a review-only run against work that already exists on `branch`.
353    ///
354    /// The expensive half of the graph is the implement wave — measured at
355    /// 111 and 134 internal tool-loop turns on this repository, against a
356    /// handful for a judge or a reviewer. The cheap half is worth running on
357    /// hand-written work too, and there was no way to reach it.
358    ///
359    /// No new state and no schema change are needed: a run with **one** viable
360    /// candidate and a tally already decided degrades `execute` to exactly
361    /// review → gate → merge, because `judge` skips a single-candidate field,
362    /// `deliberate` has fewer than two first choices to reconcile, `vote`
363    /// returns early, `tally` is already present and `fold_losers` has no
364    /// losers. Resuming such a run therefore does the right thing as well.
365    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
366        let repo = git::toplevel(repo).await?;
367        let missing = agent::missing_programs(&config.agents);
368        if !missing.is_empty() {
369            bail!(
370                "these agent programs are not on PATH: {}. Fix the roster in \
371                 magi.toml or install them.",
372                missing.join(", ")
373            );
374        }
375        if !git::branch_exists(&repo, branch).await? {
376            bail!("no branch `{branch}` in {}", repo.display());
377        }
378        let base_branch = match config.merge.base.clone() {
379            Some(b) => b,
380            None => git::current_branch(&repo)
381                .await?
382                .context("HEAD is detached; set [merge] base in magi.toml")?,
383        };
384        if base_branch == branch {
385            bail!("`{branch}` is the base branch; there is nothing to review against");
386        }
387        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
388
389        let roles = config.resolve_roles()?;
390        let max_parallel = config.graph.max_parallel.max(1);
391        // The commit subjects are the closest thing to a task statement that
392        // existing work carries, and the reviewers are told as much.
393        let log = git::log_oneline(&repo, &base_commit, branch)
394            .await
395            .unwrap_or_default();
396        let instruction = format!(
397            "Review the work already on branch `{branch}`. There is no task \
398             statement: what the change claims to do is whatever its commits \
399             say.\n\n{}",
400            if log.trim().is_empty() {
401                "(no commit messages)"
402            } else {
403                log.trim()
404            }
405        );
406        let mut state = RunState::new(
407            repo.clone(),
408            base_branch,
409            base_commit.clone(),
410            instruction,
411            config,
412        );
413
414        // An attached worktree, so the fixer's commits land on the branch under
415        // review rather than on a detached head nobody will look at again.
416        let worktree = state.worktree_root().join("under-review");
417        if let Some(parent) = worktree.parent() {
418            tokio::fs::create_dir_all(parent).await.ok();
419        }
420        let path = worktree.to_string_lossy().to_string();
421        git::git(&repo, &["worktree", "add", &path, branch])
422            .await
423            .with_context(|| {
424                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
425            })?;
426
427        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
428            .await
429            .unwrap_or(0);
430        if commits == 0 {
431            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
432        }
433        let files = git::changed_files(&worktree, &base_commit, "HEAD")
434            .await
435            .map(|f| f.len())
436            .unwrap_or(0);
437        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
438            .await
439            .unwrap_or_default();
440
441        state.candidates.push(Candidate {
442            index: 0,
443            label: 'A',
444            // Not an agent id on purpose: nothing in the roster wrote this, and
445            // the stats tables must not credit anyone with a win for it.
446            agent: "(existing branch)".to_owned(),
447            branch: branch.to_owned(),
448            worktree,
449            summary: String::new(),
450            stat,
451            files,
452            commits,
453            empty: false,
454            failed: None,
455            duration_ms: 0,
456            folded: false,
457        });
458        state.tally = Some(Tally {
459            first_choice: BTreeMap::from([('A', 0)]),
460            borda: BTreeMap::new(),
461            winner: 'A',
462            rankings: 0,
463            unanimous_initial: false,
464            deliberated: false,
465            changed_votes: 0,
466            unanimous_final: false,
467            tie_break: None,
468            // No panel sat, so no quorum applies. Zero judges is the correct
469            // number for work that never competed, and must not be reported as
470            // a collapsed panel.
471            judges: 0,
472            present: 0,
473            quorum: 0,
474            met_quorum: true,
475            uncontested: Some("review-only run: nothing competed".to_owned()),
476        });
477        state.status = RunStatus::Reviewing;
478        state.event(
479            "start",
480            format!(
481                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
482                state.id
483            ),
484        );
485        state.save()?;
486        Ok(Self {
487            state,
488            roles,
489            sem: Arc::new(Semaphore::new(max_parallel)),
490            pause: Pause::new(),
491            interrupt: Pause::new(),
492        })
493    }
494
495    /// Reopen an existing run.
496    pub fn resume(id: &str) -> Result<Self> {
497        let state = RunState::load(id)?;
498        let roles = state.config.resolve_roles()?;
499        let max_parallel = state.config.graph.max_parallel.max(1);
500        Ok(Self {
501            state,
502            roles,
503            sem: Arc::new(Semaphore::new(max_parallel)),
504            pause: Pause::new(),
505            interrupt: Pause::new(),
506        })
507    }
508
509    /// Walk the graph to a terminal state, skipping nodes already recorded.
510    pub async fn execute(&mut self) -> Result<()> {
511        // Moving again, so it is no longer parked. Set before the walk rather
512        // than in `resume`, so every way of re-entering the graph clears it
513        // and a card cannot claim a run is waiting to be resumed while the
514        // agents are already working.
515        self.state.parked = false;
516        // Any seat this state still lists as answering belongs to whatever
517        // process last drove this run — this one included, if it crashed
518        // mid-wave. Cleared and flushed immediately, before anything else
519        // runs, so a resume can never show a seat as live when nothing is
520        // asking it anything yet; the node that actually dispatches the next
521        // wave repopulates it.
522        if self.state.clear_active() {
523            self.state.save()?;
524        }
525        // A run that already lost its quorum never resumes into the verdict
526        // machinery: `deliberate` and `vote` would otherwise clobber the
527        // stalled marker back to Voting and the run would keep going past a
528        // verdict that is no longer trustworthy. Everything already recorded is
529        // kept, so the run stays resumable (or foldable) for a human to pick up.
530        //
531        // On --resume the run gets one chance to repair itself: the seats a
532        // rate limit took out are re-asked. If their quota has since reset and
533        // the quorum is restored, the run picks up and finishes; otherwise it
534        // stays stale and still-resumable for a later retry. If it does not
535        // recover, the returned status stays `Stalled` and nothing was
536        // clobbered (the recovery only mutates entries for the lost seats).
537        if self.state.status == RunStatus::Stalled {
538            if self.recover_stall().await? {
539                self.finish_after_tally().await?;
540            } else {
541                // Still below quorum: persist the marker and stay resumable.
542                self.state.save()?;
543            }
544            return Ok(());
545        }
546        // A run parked inside `land` - watching CI, mid fix-round, or
547        // waiting on the owner's merge approval - resumes directly into it,
548        // never back through `prep`. Everything before `merge` already
549        // concluded; that is the only way `status` reaches `Landing` in the
550        // first place. Re-walking `review_loop` first would also be actively
551        // wrong: its own status recomputation (see its doc) treats any
552        // clean round as reason to set `status` to `Gating`, which would
553        // clobber this marker before `merge` ever ran, and this run would
554        // never find its way back into `land` at all.
555        if self.state.status == RunStatus::Landing {
556            self.run_land().await?;
557            // `run_land` may have settled the run right here - CI came back
558            // green and the PR merged, say - without ever passing back
559            // through `merge`'s own trailing call. Whatever it left `status`
560            // as is what this has to read.
561            self.settle_questions();
562            return Ok(());
563        }
564        self.prep().await?;
565        if self.park_here()? {
566            return Ok(());
567        }
568        self.advise().await?;
569        if self.park_here()? {
570            return Ok(());
571        }
572        self.implement().await?;
573        if self.park_here()? {
574            return Ok(());
575        }
576        self.judge().await?;
577        if self.park_here()? {
578            return Ok(());
579        }
580        self.deliberate().await?;
581        if self.park_here()? {
582            return Ok(());
583        }
584        self.vote().await?;
585        if self.park_here()? {
586            return Ok(());
587        }
588        self.tally()?;
589        // A verdict that lost its quorum is not trustworthy: do not review,
590        // gate, or merge on it. Everything already done is kept, so the run
591        // stays resumable (or foldable); the human can replace the agent that
592        // ran out of quota and pick it up.
593        if self.state.status == RunStatus::Stalled {
594            // Persist the stalled marker now — the normal end-of-execute save
595            // below is below this early return, and without it a resumed run
596            // would reload a pre-tally status and keep going.
597            self.state.save()?;
598            return Ok(());
599        }
600        self.finish_after_tally().await?;
601        Ok(())
602    }
603
604    /// Park here if asked to, recording it in the run's own timeline.
605    ///
606    /// Returns whether the caller should stop walking the graph. The state is
607    /// saved either way by the node that just finished; this adds the event so
608    /// the operator's card says why a run that is neither finished nor moving
609    /// is sitting where it is.
610    fn park_here(&mut self) -> Result<bool> {
611        // Either handle asking is enough - see `Pause`'s own doc for why
612        // they are never the same one. `interrupt` is checked second so a
613        // reason it carries is preferred in the message below over a plain
614        // shutdown park racing it at the same boundary.
615        if !self.pause.parked() && !self.interrupt.parked() {
616            return Ok(false);
617        }
618        let why = match self.interrupt.reason().or_else(|| self.pause.reason()) {
619            Some(reason) => format!(
620                "parked after `{}` ({reason}) — resume to carry on from here",
621                self.state.status.as_str()
622            ),
623            None => format!(
624                "parked after `{}` — resume to carry on from here",
625                self.state.status.as_str()
626            ),
627        };
628        self.state.event("park", why);
629        self.state.parked = true;
630        self.state.save()?;
631        Ok(true)
632    }
633
634    /// Hand the runner the pause `magi serve`'s own shutdown watches.
635    pub fn on_pause(&mut self, pause: Pause) {
636        self.pause = pause;
637    }
638
639    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
640    /// scheduler asking this one run - and no other - to park so a task
641    /// marked [`crate::queue::Task::interrupt`] can run alone. See
642    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
643    /// handle.
644    pub fn watch_interrupt(&mut self, pause: Pause) {
645        self.interrupt = pause;
646    }
647
648    /// Abandon this run's own open questions, once `status` has actually
649    /// settled rather than merely paused.
650    ///
651    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
652    /// either back up with the candidates, the review round and the seat
653    /// sessions already on disk, so a question an implementer asked mid-round
654    /// may still get a real answer read by a real resume. Only the three
655    /// statuses `resumable` excludes are actually final: the run merged, or
656    /// it reached `Ready` with nothing left to do, or it failed outright with
657    /// no established point to continue from. In every one of those the seat
658    /// that asked is gone for good, exactly like the run being deleted under
659    /// `magi run rm` - so the same cleanup applies, worded for what actually
660    /// happened instead of "the run was deleted".
661    ///
662    /// Best-effort and silent on success: called from every place `status`
663    /// can land on one of those three, including ones a resumed run revisits,
664    /// so it must cost nothing when there was nothing open to begin with.
665    fn settle_questions(&mut self) {
666        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
667            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
668        }
669    }
670
671    /// The tail of the graph after a trustworthy tally: fold losers, review,
672    /// gate, merge, and persist.
673    async fn finish_after_tally(&mut self) -> Result<()> {
674        self.fold_losers().await?;
675        // Before review starts, and again right before the gate: a run's
676        // review rounds can themselves take long enough for the base to move
677        // a second time, and the gate is the one node whose "green" gets
678        // acted on.
679        self.sync_to_base().await?;
680        self.review_loop().await?;
681        self.sync_to_base().await?;
682        self.gate().await?;
683        self.merge().await?;
684        self.state.save()?;
685        Ok(())
686    }
687
688    // ---------------------------------------------------------------- prep
689
690    async fn prep(&mut self) -> Result<()> {
691        if !self.state.candidates.is_empty() {
692            return Ok(());
693        }
694        self.state.status = RunStatus::Prep;
695        let repo = self.state.repo.clone();
696        let base = self.state.base_commit.clone();
697        let root = self.state.worktree_root();
698        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
699
700        // The hook is the write-time half of the blindness contract; the
701        // presentation filter in `blind` is the half that cannot be bypassed.
702        let hooks_dir = self.state.dir().join("hooks");
703        if self.state.config.blind.commit_msg_hook {
704            std::fs::create_dir_all(&hooks_dir)
705                .with_context(|| format!("create {}", hooks_dir.display()))?;
706            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
707            let path = hooks_dir.join("commit-msg");
708            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
709            make_executable(&path)?;
710            // Ref-counted rather than a plain idempotent set: with more than
711            // one run able to be in flight in the same repository at once
712            // (see `Config::daemon.max_concurrent_runs`), a bare "already
713            // true?" check cannot tell "another run of mine still needs
714            // this" from "nobody does", and the run that happens to finish
715            // first would disable the hook out from under a sibling still
716            // relying on it.
717            git::acquire_worktree_config(&repo).await?;
718            self.state.enabled_worktree_config = true;
719        }
720
721        for (index, (spec, label)) in self
722            .roles
723            .implementers
724            .clone()
725            .into_iter()
726            .zip(labels)
727            .enumerate()
728        {
729            let branch = self.state.branch_for(label);
730            let worktree = root.join(format!("cand-{label}"));
731            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
732            if self.state.config.blind.commit_msg_hook {
733                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
734            }
735            git::local_exclude(&worktree, "/.magi/").await?;
736            self.state.candidates.push(Candidate {
737                index,
738                label,
739                agent: spec.id.clone(),
740                branch,
741                worktree,
742                summary: String::new(),
743                stat: String::new(),
744                files: 0,
745                commits: 0,
746                empty: false,
747                failed: None,
748                duration_ms: 0,
749                folded: false,
750            });
751        }
752
753        for j in 1..=self.roles.judges.len() {
754            let wt = root.join(format!("judge-{j}"));
755            if !wt.exists() {
756                git::worktree_add_detached(&repo, &wt, &base).await?;
757            }
758        }
759
760        // Disposable, detached checkouts for the design-deliberation stage's
761        // advisor seats — the same shape as the judges' above, at the same
762        // base commit, since advisors also only ever read. Sized off the
763        // configured count directly rather than a resolved roster: unlike
764        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
765        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
766        // `prep` has no `ResolvedRoles` field to read a count from here.
767        if self.state.config.graph.advise {
768            for k in 1..=self.state.config.graph.advisors {
769                let wt = root.join(format!("advisor-{k}"));
770                if !wt.exists() {
771                    git::worktree_add_detached(&repo, &wt, &base).await?;
772                }
773            }
774        }
775
776        // A judge cannot tell it is looking at its own patch — the seats keep
777        // separate conversations — but a panel that shares agents with the
778        // field is less independent than it looks, and that is worth saying out
779        // loud once per run rather than leaving it in the config.
780        let authors: Vec<&str> = self
781            .roles
782            .implementers
783            .iter()
784            .map(|a| a.id.as_str())
785            .collect();
786        let overlap: Vec<String> = self
787            .roles
788            .judges
789            .iter()
790            .enumerate()
791            .filter(|(_, j)| authors.contains(&j.id.as_str()))
792            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
793            .collect();
794        if !overlap.is_empty() {
795            let note = format!(
796                "{} also authored a candidate; blind, but the panel is less \
797                 independent than {} distinct agents would be",
798                overlap.join(", "),
799                self.roles.judges.len()
800            );
801            self.state.event("prep", note);
802        }
803
804        self.state.event(
805            "prep",
806            format!(
807                "{} candidates, {} judges, base {} ({})",
808                self.state.candidates.len(),
809                self.roles.judges.len(),
810                &self.state.base_commit[..7.min(self.state.base_commit.len())],
811                self.state.base_branch
812            ),
813        );
814        self.state.status = RunStatus::Implementing;
815        self.state.save()?;
816        Ok(())
817    }
818
819    // -------------------------------------------------------------- advise
820
821    /// The design-deliberation stage: independent, read-only advisor seats
822    /// each sketch a design before any implementer touches the repository,
823    /// and (when at least one produced a usable proposal) a synthesis seat
824    /// blends them into a brief `implement` carries in every candidate's
825    /// prompt.
826    ///
827    /// `[graph] advise` is the on/off switch, on by default; `[graph]
828    /// advisors` is the proposal count. Everything here is best-effort and
829    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
830    /// that cannot reach quota, or a synthesis seat that produced nothing
831    /// usable all leave `implement` exactly as it was before this stage
832    /// existed — the task instruction alone — rather than failing the whole
833    /// competition over an enrichment stage. Every outcome is still recorded
834    /// as an event, so a run that got nothing from this stage says why.
835    ///
836    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
837    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
838    /// resumed run whose stage failed would re-run it, and re-spend the
839    /// agent calls, on every reentry before `implement`.
840    ///
841    /// Also skipped once any candidate shows implementation progress — the
842    /// exact predicate `implement` itself uses to decide a candidate is no
843    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
844    /// is not enough: a run created by an older binary that predates this
845    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
846    /// an already-`Implementing`-or-later run under this build would
847    /// otherwise walk straight back through `prep` (a no-op once candidates
848    /// exist) into this node and spawn every advisor seat against worktrees
849    /// `prep` never recreated — after implementation has already started,
850    /// which is exactly the invariant this stage exists to guarantee.
851    async fn advise(&mut self) -> Result<()> {
852        let implement_untouched = self
853            .state
854            .candidates
855            .iter()
856            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
857        if !self.state.config.graph.advise || self.state.advise_attempted {
858            return Ok(());
859        }
860        if !implement_untouched {
861            self.state.event(
862                "advise",
863                "skipping the design-deliberation stage: at least one \
864                 candidate already shows implementation progress, so this \
865                 run is past the point the stage exists to run before"
866                    .to_owned(),
867            );
868            self.state.advise_attempted = true;
869            self.state.save()?;
870            return Ok(());
871        }
872        let run_id = self.state.id.clone();
873        let prompts = self.state.config.prompts.clone();
874        let instruction = self.state.instruction.clone();
875        let language = self.state.config.graph.language.clone();
876        let root = self.state.worktree_root();
877        let n = self.state.config.graph.advisors;
878        let where_recorded = self.state.dir().join("run.json");
879
880        let seats = match self.state.config.advisors() {
881            Ok(seats) if !seats.is_empty() => seats,
882            Ok(_) => {
883                self.state.event(
884                    "advise",
885                    format!(
886                        "[graph] advisors is 0; skipping the design-deliberation \
887                         stage and continuing without a synthesis brief (see {})",
888                        where_recorded.display()
889                    ),
890                );
891                self.state.advise_attempted = true;
892                self.state.save()?;
893                return Ok(());
894            }
895            Err(e) => {
896                self.state.event(
897                    "advise",
898                    format!(
899                        "could not resolve advisor seats ({e:#}); continuing \
900                         without a design-deliberation brief (see {})",
901                        where_recorded.display()
902                    ),
903                );
904                self.state.advise_attempted = true;
905                self.state.save()?;
906                return Ok(());
907            }
908        };
909
910        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
911        let artifacts = agent::artifacts_dir(&self.state.dir());
912        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
913
914        let mut jobs = Vec::new();
915        for (i, spec) in seats.iter().cloned().enumerate() {
916            let seat_key = format!("advisor-{}", i + 1);
917            let seat = self.seat(&seat_key, &spec.id);
918            jobs.push(SeatJob {
919                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
920                spec,
921                seat,
922                cwd: worktrees[i % worktrees.len()].clone(),
923                timeout,
924                allow_write: false,
925                sessions: false,
926                artifacts: artifacts.clone(),
927                stem: seat_key,
928            });
929        }
930
931        self.state.event(
932            "advise",
933            format!(
934                "{} advisor seat(s) sketching a design in parallel",
935                jobs.len()
936            ),
937        );
938        let mut quota_losses = Vec::new();
939        let cache = self.state.config.cache_dir();
940        let ctx = WaveCtx {
941            run: &run_id,
942            node: "advise",
943            prompts: &prompts,
944            cache: cache.as_deref(),
945        };
946        let results = ask_json_wave::<Proposal>(
947            jobs,
948            Arc::clone(&self.sem),
949            self.state.config.graph.retries,
950            &ctx,
951            &mut quota_losses,
952            &mut self.state,
953            &|p: &Proposal| p.validate(),
954        )
955        .await;
956        self.state.quota.extend(quota_losses);
957
958        let mut records = Vec::with_capacity(results.len());
959        for (i, (seat, res)) in results.into_iter().enumerate() {
960            let agent_id = seat.agent.clone();
961            self.state.seats.insert(seat.key.clone(), seat);
962            match res {
963                Ok((proposal, out)) => {
964                    self.state
965                        .event("advise", format!("advisor-{} proposed a design", i + 1));
966                    records.push(advise::AdvisorRecord::proposed(
967                        i + 1,
968                        agent_id,
969                        proposal,
970                        out.duration_ms,
971                    ));
972                }
973                Err(e) => {
974                    self.state.event(
975                        "advise",
976                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
977                    );
978                    records.push(advise::AdvisorRecord::failed(
979                        i + 1,
980                        agent_id,
981                        e.to_string(),
982                    ));
983                }
984            }
985        }
986
987        let mut advice = advise::Advice {
988            records,
989            synthesis: None,
990        };
991        if advice.proposals().is_empty() {
992            self.state.event(
993                "advise",
994                "no advisor produced a usable proposal; continuing without a \
995                 synthesis brief"
996                    .to_owned(),
997            );
998        } else {
999            match self
1000                .synthesize_brief(
1001                    &advice,
1002                    &instruction,
1003                    &language,
1004                    &worktrees[0],
1005                    &artifacts,
1006                    &run_id,
1007                    &prompts,
1008                    cache.as_deref(),
1009                )
1010                .await
1011            {
1012                Ok(Some(text)) => {
1013                    self.state.event(
1014                        "advise",
1015                        "synthesized a design brief for the implementer".to_owned(),
1016                    );
1017                    advice.synthesis = Some(text);
1018                }
1019                Ok(None) => {
1020                    self.state.event(
1021                        "advise",
1022                        "the synthesis seat produced nothing usable; continuing \
1023                         without a design brief"
1024                            .to_owned(),
1025                    );
1026                }
1027                Err(e) => {
1028                    self.state.event(
1029                        "advise",
1030                        format!("could not synthesize a design brief: {e:#}"),
1031                    );
1032                }
1033            }
1034        }
1035        advise::apply_reflection(&mut advice);
1036
1037        self.state.advice = Some(advice);
1038        self.state.advise_attempted = true;
1039        self.state.save()?;
1040        Ok(())
1041    }
1042
1043    /// The synthesis seat: reads every advisor's proposal and blends them
1044    /// into the design brief `advise` stores on [`RunState::advice`]. Split
1045    /// out of [`Runner::advise`] only for readability — it is not called
1046    /// anywhere else.
1047    ///
1048    /// Picked the same way [`crate::talk`]'s standing conversation and
1049    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`] with no
1050    /// explicit id, rather than a dedicated `[roles]` entry — one more role
1051    /// to configure for a seat that runs once per run and, unlike the
1052    /// advisors it reads, never needs more than one.
1053    #[allow(clippy::too_many_arguments)]
1054    async fn synthesize_brief(
1055        &mut self,
1056        advice: &advise::Advice,
1057        instruction: &str,
1058        language: &str,
1059        cwd: &Path,
1060        artifacts: &Path,
1061        run_id: &str,
1062        prompts: &Prompts,
1063        cache: Option<&Path>,
1064    ) -> Result<Option<String>> {
1065        let spec = agent::pick(&self.state.config.agents, None, &agent::installed)?;
1066        let mut seat = self.seat("advise-synthesis", &spec.id);
1067        let proposals = advice.proposals();
1068        let mut prompt = prompt::with_overlay(
1069            prompt::synthesize_brief(instruction, &proposals, language),
1070            prompts.overlay("advise"),
1071        );
1072        if cache.is_some() {
1073            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
1074            // below — see `prompt::build_cache_note`'s doc for why telling a
1075            // read-only seat to build through the shared cache is exactly how
1076            // a sandbox's write refusal gets misread as a defect.
1077            prompt.push('\n');
1078            prompt.push_str(&prompt::build_cache_note("advise", false));
1079        }
1080        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
1081        let out = agent::invoke(
1082            &spec,
1083            &mut seat,
1084            &Invocation {
1085                cwd,
1086                prompt: &prompt,
1087                timeout,
1088                allow_write: false,
1089                sessions: false,
1090                artifacts,
1091                stem: "advise-synthesis",
1092                run: run_id,
1093                node: "advise",
1094                cache_dir: None,
1095                attachments: &[],
1096            },
1097        )
1098        .await?;
1099        self.state.seats.insert(seat.key.clone(), seat);
1100        if !out.usable() {
1101            return Ok(None);
1102        }
1103        let text =
1104            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
1105        Ok((!text.trim().is_empty()).then_some(text))
1106    }
1107
1108    // ----------------------------------------------------------- implement
1109
1110    async fn implement(&mut self) -> Result<()> {
1111        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1112        // agent files with `magi task add` name the run that paid for it. The
1113        // prompt overlay is cloned alongside it because the waves borrow it
1114        // while `self` is mutably borrowed by the node's own bookkeeping.
1115        let run_id = self.state.id.clone();
1116        let prompts = self.state.config.prompts.clone();
1117        let todo: Vec<usize> = self
1118            .state
1119            .candidates
1120            .iter()
1121            .enumerate()
1122            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1123            .map(|(i, _)| i)
1124            .collect();
1125        if todo.is_empty() {
1126            return self.after_implement();
1127        }
1128        self.state.status = RunStatus::Implementing;
1129
1130        let language = self.state.config.graph.language.clone();
1131        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1132        let sessions = self.state.config.graph.sessions;
1133        let artifacts = agent::artifacts_dir(&self.state.dir());
1134        // The design-deliberation stage's blended brief, when `advise` found
1135        // one — carried into every implementer's prompt the same way
1136        // regardless of which candidate it is.
1137        let brief = self
1138            .state
1139            .advice
1140            .as_ref()
1141            .and_then(|a| a.synthesis.as_deref())
1142            .map(str::to_owned);
1143
1144        let mut jobs = Vec::new();
1145        for &i in &todo {
1146            let (index, label, worktree) = {
1147                let c = &self.state.candidates[i];
1148                (c.index, c.label, c.worktree.clone())
1149            };
1150            let spec = self.roles.implementers[index].clone();
1151            let seat_key = format!("impl-{label}");
1152            let seat = self.seat(&seat_key, &spec.id);
1153            let instruction = self.state.instruction.clone();
1154            jobs.push(SeatJob {
1155                spec,
1156                seat,
1157                prompt: prompt::implement(
1158                    &instruction,
1159                    &worktree.to_string_lossy(),
1160                    &language,
1161                    brief.as_deref(),
1162                ),
1163                cwd: worktree,
1164                timeout,
1165                allow_write: true,
1166                sessions,
1167                artifacts: artifacts.clone(),
1168                stem: format!("impl-{label}"),
1169            });
1170        }
1171
1172        self.state.event(
1173            "implement",
1174            format!("{} candidates in parallel", jobs.len()),
1175        );
1176        // Kept so a seat whose CLI hung up can be asked again from the same
1177        // job: `wave` consumes what it is given.
1178        let sent = jobs.clone();
1179        let cache = self.state.config.cache_dir();
1180        let ctx = WaveCtx {
1181            run: &run_id,
1182            node: "implement",
1183            prompts: &prompts,
1184            cache: cache.as_deref(),
1185        };
1186        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1187        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1188            .await;
1189        self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
1190            .await;
1191
1192        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1193            let seat_key = seat.key.clone();
1194            self.state.seats.insert(seat.key.clone(), seat);
1195            let label = self.state.candidates[i].label;
1196            let worktree = self.state.candidates[i].worktree.clone();
1197            let base = self.state.base_commit.clone();
1198
1199            let (summary, duration, failed) = match out {
1200                AgentOutcome::Ok(o) => {
1201                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1202                    let failed = (!o.usable()).then(|| {
1203                        if o.timed_out {
1204                            "agent timed out".to_owned()
1205                        } else {
1206                            format!("agent exited with {:?}", o.exit_code)
1207                        }
1208                    });
1209                    (text, o.duration_ms, failed)
1210                }
1211                // Left un-resumed by `resume_undelivered` (a dirty tree
1212                // already rescues the work, or there was no session left to
1213                // resume into) — reported like the ordinary failure it is,
1214                // never as if `o.text` (the CLI's raw error JSON) were an
1215                // answer.
1216                AgentOutcome::Dropped(o) => {
1217                    let why = o
1218                        .dropped
1219                        .as_ref()
1220                        .map(|d| d.why.as_str())
1221                        .unwrap_or("the CLI ended the stream without delivering its answer");
1222                    (
1223                        String::new(),
1224                        o.duration_ms,
1225                        Some(format!("the CLI dropped the stream ({why})")),
1226                    )
1227                }
1228                AgentOutcome::Quota(o) => {
1229                    self.state.quota.push(QuotaLoss {
1230                        seat: seat_key,
1231                        node: "implement".to_owned(),
1232                        at: Timestamp::now(),
1233                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1234                    });
1235                    (
1236                        String::new(),
1237                        o.duration_ms,
1238                        Some("rate limited (quota); produced no change".to_owned()),
1239                    )
1240                }
1241                AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
1242            };
1243
1244            // Rescue anything the agent edited but never committed: an
1245            // uncommitted candidate would silently be an empty one.
1246            let rescued = git::commit_all(
1247                &worktree,
1248                &format!("magi: candidate {label} (uncommitted work)"),
1249            )
1250            .await
1251            .unwrap_or(false);
1252            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1253                .await
1254                .unwrap_or(0);
1255            let patch = git::diff(&worktree, &base, "HEAD")
1256                .await
1257                .unwrap_or_default();
1258            let stat = git::diff_stat(&worktree, &base, "HEAD")
1259                .await
1260                .unwrap_or_default();
1261            let files = git::changed_files(&worktree, &base, "HEAD")
1262                .await
1263                .map(|f| f.len())
1264                .unwrap_or(0);
1265            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1266
1267            let c = &mut self.state.candidates[i];
1268            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1269            c.stat = stat;
1270            c.files = files;
1271            c.commits = commits;
1272            c.duration_ms = duration;
1273            c.empty = commits == 0 || patch.trim().is_empty();
1274            // An agent that failed but still produced a committed change stays
1275            // in the running: the patch is what gets judged, not the exit code.
1276            c.failed = match failed {
1277                Some(_) if c.empty => failed,
1278                _ => None,
1279            };
1280            let note = match (&c.failed, c.empty, rescued) {
1281                (Some(e), _, _) => format!("candidate {label}: {e}"),
1282                (None, true, _) => format!("candidate {label}: no change produced"),
1283                (None, false, true) => {
1284                    format!(
1285                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1286                    )
1287                }
1288                (None, false, false) => {
1289                    format!("candidate {label}: {files} files, {commits} commits")
1290                }
1291            };
1292            self.state.event("implement", note);
1293            self.state.save()?;
1294        }
1295
1296        self.after_implement()
1297    }
1298
1299    /// Ask again, once, for work a CLI did and then failed to hand over.
1300    ///
1301    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1302    /// status with an empty response and a usage report showing output tokens,
1303    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1304    /// seven minutes and 14,267 output tokens that arrived as an empty
1305    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1306    ///
1307    /// Two conditions, and both matter:
1308    ///
1309    /// - **Only when the tree is untouched.** Often the agent has already
1310    ///   written its files and only the closing message was lost; the rescue
1311    ///   commit below picks that up and there is nothing to ask for. Re-asking
1312    ///   then would pay for a second implementation of work already on disk.
1313    /// - **Once.** A CLI that drops one stream can drop the next, and this
1314    ///   node is the most expensive in the graph.
1315    ///
1316    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1317    /// dropped reply still carried its `conversation_id`, so the seat is asked
1318    /// to finish what it was doing rather than sent the whole task again. It
1319    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1320    /// node's - for the same reason a re-ranked judge does: restating finished
1321    /// work is not the work.
1322    ///
1323    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1324    /// same way until it resets, while an abandoned conversation is still
1325    /// there to be picked up.
1326    async fn resume_undelivered(
1327        &mut self,
1328        results: &mut [(usize, SeatState, AgentOutcome)],
1329        sent: &[SeatJob],
1330        prompts: &Prompts,
1331        run_id: &str,
1332    ) {
1333        for (wi, seat, out) in results.iter_mut() {
1334            let Some(dropped) = (match &*out {
1335                AgentOutcome::Dropped(o) => o.dropped.clone(),
1336                _ => None,
1337            }) else {
1338                continue;
1339            };
1340            let Some(job) = sent.get(*wi) else { continue };
1341            // Already on disk? Then only the closing message was lost.
1342            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1343                self.state.event(
1344                    "implement",
1345                    format!(
1346                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1347                         work is in the tree",
1348                        seat.key, dropped.output_tokens, dropped.why
1349                    ),
1350                );
1351                continue;
1352            }
1353            // The re-ask only makes sense as a resume: `resume_after_drop`
1354            // says nothing about the task, trusting the seat to still hold it.
1355            // Without a session to resume — sessions disabled, or this CLI's
1356            // drop shape happened not to carry a session id — that prompt
1357            // would open a brand-new conversation with no context at all,
1358            // which is worse than leaving this as the ordinary failure it
1359            // already is.
1360            if !has_context(&job.spec, seat, job.sessions) {
1361                self.state.event(
1362                    "implement",
1363                    format!(
1364                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1365                         is no session left to resume",
1366                        seat.key, dropped.output_tokens, dropped.why
1367                    ),
1368                );
1369                continue;
1370            }
1371            self.state.event(
1372                "implement",
1373                format!(
1374                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1375                     conversation",
1376                    seat.key, dropped.output_tokens, dropped.why
1377                ),
1378            );
1379            let mut retry = job.clone();
1380            retry.seat = seat.clone();
1381            retry.prompt = prompt::resume_after_drop(&dropped.why);
1382            retry.timeout = retry_budget(job.timeout, true);
1383            retry.stem = format!("{}-resume", job.stem);
1384            let cache = self.state.config.cache_dir();
1385            let ctx = WaveCtx {
1386                run: run_id,
1387                node: "implement",
1388                prompts,
1389                cache: cache.as_deref(),
1390            };
1391            let (resumed_seat, resumed) =
1392                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1393            *seat = resumed_seat;
1394            *out = resumed;
1395        }
1396    }
1397
1398    /// Ask an implement seat's own CLI to confirm what it started, once, when
1399    /// its reply reported a command whose completion status it never
1400    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
1401    /// that does and does not mean.
1402    ///
1403    /// The completion contract this task asks for, extended to `implement`
1404    /// with the same signal `continue_fix_report` reads for the fixer,
1405    /// rather than a keyword search over the reply or a hard requirement on
1406    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
1407    /// a candidate's CLI turn ended cleanly while a test run it had started
1408    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
1409    /// named in it at all is untouched by this: `commands` is empty, so
1410    /// there is nothing to be unconfirmed.
1411    ///
1412    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
1413    /// this is not about recovering edits that might already be on disk, it
1414    /// is about a result the seat itself never vouched for, which resuming
1415    /// asks for regardless of what the tree already holds. Bounded to one
1416    /// attempt for the same reason `resume_undelivered` is — this is the
1417    /// most expensive node in the graph — and a seat that still cannot
1418    /// confirm on that attempt is left as whatever its (possibly still
1419    /// unconfirmed) reply says; this does not invent a new "failed" reason
1420    /// for a candidate that otherwise produced a real, committed change.
1421    async fn resume_unconfirmed_commands(
1422        &mut self,
1423        results: &mut [(usize, SeatState, AgentOutcome)],
1424        sent: &[SeatJob],
1425        prompts: &Prompts,
1426        run_id: &str,
1427    ) {
1428        for (wi, seat, out) in results.iter_mut() {
1429            let AgentOutcome::Ok(o) = &*out else {
1430                continue;
1431            };
1432            if !has_unconfirmed_command(&o.commands) {
1433                continue;
1434            }
1435            let Some(job) = sent.get(*wi) else { continue };
1436            if !has_context(&job.spec, seat, job.sessions) {
1437                self.state.event(
1438                    "implement",
1439                    format!(
1440                        "{}: the reply named a command whose own CLI never confirmed the exit \
1441                         status of, but there is no session left to resume",
1442                        seat.key
1443                    ),
1444                );
1445                continue;
1446            }
1447            self.state.event(
1448                "implement",
1449                format!(
1450                    "{}: the reply named a command whose own CLI never confirmed the exit \
1451                     status of; resuming the conversation",
1452                    seat.key
1453                ),
1454            );
1455            let mut retry = job.clone();
1456            retry.seat = seat.clone();
1457            retry.prompt = prompt::resume_incomplete(
1458                "a command in your last reply had no confirmed exit status",
1459            );
1460            retry.timeout = retry_budget(job.timeout, true);
1461            retry.stem = format!("{}-confirm", job.stem);
1462            let cache = self.state.config.cache_dir();
1463            let ctx = WaveCtx {
1464                run: run_id,
1465                node: "implement",
1466                prompts,
1467                cache: cache.as_deref(),
1468            };
1469            let (resumed_seat, resumed) =
1470                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1471            *seat = resumed_seat;
1472            *out = resumed;
1473        }
1474    }
1475
1476    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
1477    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
1478    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
1479    /// that motivated this.
1480    ///
1481    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
1482    /// own nudge loop already covers for judge/review/vote seats, and not a
1483    /// dropped stream, which [`Runner::resume_undelivered`] covers for
1484    /// implement seats: here the CLI turn genuinely finished while the node's
1485    /// own work — the fixer's account of what it did — had not. Gated purely
1486    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
1487    /// reply, never on any wording in it, so a fixer whose valid, first-try
1488    /// `FixReport` happens to mention having waited on a background test is
1489    /// never resumed — the `Ok(report)` branch at the call site returns
1490    /// before this is ever invoked.
1491    ///
1492    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
1493    /// attempt ([`retry_budget`]), nothing attempted once the session is
1494    /// gone, and a quota hit ends the loop immediately rather than retrying a
1495    /// rate limit that fails the same way again.
1496    async fn continue_fix_report(
1497        &mut self,
1498        mut seat: SeatState,
1499        parse_err: String,
1500        job: &SeatJob,
1501        prompts: &Prompts,
1502        run_id: &str,
1503        round: usize,
1504    ) -> (
1505        SeatState,
1506        Option<FixReport>,
1507        Option<String>,
1508        ContinuationRecord,
1509    ) {
1510        let mut last_err = parse_err;
1511        let mut cumulative_wait_ms = 0u64;
1512        let mut attempts = 0usize;
1513        loop {
1514            if !has_context(&job.spec, &seat, job.sessions) {
1515                self.state.event(
1516                    "fix",
1517                    format!(
1518                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
1519                         session left to resume into"
1520                    ),
1521                );
1522                let outcome = if attempts == 0 {
1523                    ContinuationOutcome::NoSession
1524                } else {
1525                    ContinuationOutcome::Exhausted
1526                };
1527                return (
1528                    seat,
1529                    None,
1530                    Some(format!("unparsable fix report: {last_err}")),
1531                    ContinuationRecord {
1532                        attempts,
1533                        cumulative_wait_ms,
1534                        outcome,
1535                    },
1536                );
1537            }
1538            if attempts >= MAX_FIX_CONTINUATIONS {
1539                self.state.event(
1540                    "fix",
1541                    format!(
1542                        "round {round}: fixer's reply still had no adoption report after \
1543                         {attempts} continuation(s) ({last_err}); giving up"
1544                    ),
1545                );
1546                return (
1547                    seat,
1548                    None,
1549                    Some(format!(
1550                        "unparsable fix report after {attempts} continuation(s): {last_err}"
1551                    )),
1552                    ContinuationRecord {
1553                        attempts,
1554                        cumulative_wait_ms,
1555                        outcome: ContinuationOutcome::Exhausted,
1556                    },
1557                );
1558            }
1559            attempts += 1;
1560            self.state.event(
1561                "fix",
1562                format!(
1563                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
1564                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
1565                ),
1566            );
1567            let mut retry = job.clone();
1568            retry.seat = seat.clone();
1569            retry.prompt = prompt::resume_incomplete(&last_err);
1570            retry.timeout = retry_budget(job.timeout, true);
1571            retry.stem = format!("{}-continue{attempts}", job.stem);
1572            let cache = self.state.config.cache_dir();
1573            let ctx = WaveCtx {
1574                run: run_id,
1575                node: "fix",
1576                prompts,
1577                cache: cache.as_deref(),
1578            };
1579            let (resumed_seat, resumed_out) = run_one(
1580                retry,
1581                Arc::clone(&self.sem),
1582                &ctx,
1583                &mut self.state,
1584                attempts,
1585            )
1586            .await;
1587            seat = resumed_seat;
1588            match resumed_out {
1589                AgentOutcome::Ok(o) => {
1590                    cumulative_wait_ms += o.duration_ms;
1591                    match verdict::extract_json::<FixReport>(&o.text) {
1592                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
1593                            self.state.event(
1594                                "fix",
1595                                format!(
1596                                    "round {round}: fixer's adoption report recovered after \
1597                                     {attempts} continuation(s)"
1598                                ),
1599                            );
1600                            return (
1601                                seat,
1602                                Some(report),
1603                                None,
1604                                ContinuationRecord {
1605                                    attempts,
1606                                    cumulative_wait_ms,
1607                                    outcome: ContinuationOutcome::Resumed,
1608                                },
1609                            );
1610                        }
1611                        // The report parsed, but this same reply's own
1612                        // CommandEvidence — the identical record `state.jobs`
1613                        // renders — names a command whose CLI never
1614                        // confirmed an exit status. Read together, that is
1615                        // not a resolved answer: keep nudging rather than
1616                        // accept a report standing next to a command the
1617                        // seat's own CLI cannot vouch for.
1618                        Ok(_) => {
1619                            last_err = "the reply parsed, but it reported a command whose own CLI \
1620                                 never confirmed an exit status"
1621                                .to_owned();
1622                        }
1623                        Err(e) => last_err = e.to_string(),
1624                    }
1625                }
1626                AgentOutcome::Quota(o) => {
1627                    cumulative_wait_ms += o.duration_ms;
1628                    self.state.quota.push(QuotaLoss {
1629                        seat: seat.key.clone(),
1630                        node: "fix".to_owned(),
1631                        at: Timestamp::now(),
1632                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1633                    });
1634                    self.state.event(
1635                        "fix",
1636                        format!(
1637                            "round {round}: continuation rate limited (quota); not retrying now"
1638                        ),
1639                    );
1640                    return (
1641                        seat,
1642                        None,
1643                        Some("rate limited (quota) while recovering the fix report".to_owned()),
1644                        ContinuationRecord {
1645                            attempts,
1646                            cumulative_wait_ms,
1647                            outcome: ContinuationOutcome::QuotaLost,
1648                        },
1649                    );
1650                }
1651                AgentOutcome::Dropped(o) => {
1652                    cumulative_wait_ms += o.duration_ms;
1653                    let why = o
1654                        .dropped
1655                        .as_ref()
1656                        .map(|d| d.why.as_str())
1657                        .unwrap_or("the CLI ended the stream without delivering its answer");
1658                    last_err = format!("the CLI dropped the stream ({why})");
1659                }
1660                AgentOutcome::Failed(e) => last_err = e,
1661            }
1662        }
1663    }
1664
1665    fn after_implement(&mut self) -> Result<()> {
1666        // Scan every candidate patch once the set is complete.
1667        if self.state.leaks.is_empty() {
1668            let cfg = self.state.config.blind.clone();
1669            let mut leaks = Vec::new();
1670            for c in &self.state.candidates {
1671                let Some(patch) =
1672                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
1673                else {
1674                    continue;
1675                };
1676                leaks.extend(blind::scan(
1677                    &format!("candidate {} patch", c.label),
1678                    &patch,
1679                    &cfg.vendor_tokens,
1680                ));
1681            }
1682            if !leaks.is_empty() {
1683                let summary = leaks
1684                    .iter()
1685                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
1686                    .collect::<Vec<_>>()
1687                    .join(", ");
1688                match cfg.on_leak {
1689                    LeakPolicy::Fail => {
1690                        self.state.status = RunStatus::Failed;
1691                        self.state
1692                            .event("blind", format!("vendor text in a patch: {summary}"));
1693                        self.state.leaks = leaks;
1694                        self.state.save()?;
1695                        self.settle_questions();
1696                        bail!(
1697                            "blind.on_leak = \"fail\" and vendor text reached a \
1698                             judged patch: {summary}"
1699                        );
1700                    }
1701                    LeakPolicy::Redact => self.state.event(
1702                        "blind",
1703                        format!("redacting vendor text for judging: {summary}"),
1704                    ),
1705                    LeakPolicy::Warn => self.state.event(
1706                        "blind",
1707                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
1708                    ),
1709                }
1710                self.state.leaks = leaks;
1711            }
1712        }
1713
1714        if self.state.viable().is_empty() {
1715            self.state.status = RunStatus::Failed;
1716            self.state.save()?;
1717            self.settle_questions();
1718            bail!("no candidate produced a change; nothing to judge");
1719        }
1720        self.state.status = RunStatus::Judging;
1721        self.state.save()?;
1722        Ok(())
1723    }
1724
1725    // --------------------------------------------------------------- judge
1726
1727    async fn judge(&mut self) -> Result<()> {
1728        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1729        // agent files with `magi task add` name the run that paid for it. The
1730        // prompt overlay is cloned alongside it because the waves borrow it
1731        // while `self` is mutably borrowed by the node's own bookkeeping.
1732        let run_id = self.state.id.clone();
1733        let prompts = self.state.config.prompts.clone();
1734        if !self.state.judgements.is_empty() || self.state.judge_skipped {
1735            return Ok(());
1736        }
1737        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1738        if viable.len() == 1 {
1739            // Recorded so this is a one-time event: `judgements` stays empty
1740            // either way, which without this flag is indistinguishable from
1741            // "not yet judged" on the next reentry — and status is left
1742            // untouched, so a later node's conclusion (e.g. `Blocked` after
1743            // the review budget ran out) survives a resume instead of being
1744            // clobbered back to `Judging` by this node running again.
1745            self.state.judge_skipped = true;
1746            self.state.event(
1747                "judge",
1748                format!(
1749                    "only candidate {} produced a change; judging skipped",
1750                    viable[0].label
1751                ),
1752            );
1753            self.state.save()?;
1754            return Ok(());
1755        }
1756        self.state.status = RunStatus::Judging;
1757
1758        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1759        let language = self.state.config.graph.language.clone();
1760        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1761        let sessions = self.state.config.graph.sessions;
1762        let artifacts = agent::artifacts_dir(&self.state.dir());
1763        let root = self.state.worktree_root();
1764        let base_short = short(&self.state.base_commit);
1765
1766        let mut jobs = Vec::new();
1767        let mut orders = Vec::new();
1768        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1769            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1770            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1771            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
1772            let seat_key = format!("judge-{}", j + 1);
1773            let seat = self.seat(&seat_key, &spec.id);
1774            jobs.push(SeatJob {
1775                prompt: prompt::judge(
1776                    &self.state.instruction,
1777                    &views,
1778                    self.roles.judges.len(),
1779                    &base_short,
1780                    &language,
1781                ),
1782                spec,
1783                seat,
1784                cwd: root.join(format!("judge-{}", j + 1)),
1785                timeout,
1786                allow_write: false,
1787                sessions,
1788                artifacts: artifacts.clone(),
1789                stem: format!("judge-{}", j + 1),
1790            });
1791        }
1792
1793        self.state.event(
1794            "judge",
1795            format!(
1796                "{} judges ranking {} candidates blind",
1797                jobs.len(),
1798                viable.len()
1799            ),
1800        );
1801        let labels_for_check = labels.clone();
1802        let mut quota_losses = Vec::new();
1803        let cache = self.state.config.cache_dir();
1804        let ctx = WaveCtx {
1805            run: &run_id,
1806            node: "judge",
1807            prompts: &prompts,
1808            cache: cache.as_deref(),
1809        };
1810        let results = ask_json_wave::<Ranking>(
1811            jobs,
1812            Arc::clone(&self.sem),
1813            self.state.config.graph.retries,
1814            &ctx,
1815            &mut quota_losses,
1816            &mut self.state,
1817            &move |r: &Ranking| r.validate(&labels_for_check),
1818        )
1819        .await;
1820        self.state.quota.extend(quota_losses);
1821
1822        for (j, (seat, res)) in results.into_iter().enumerate() {
1823            let agent_id = seat.agent.clone();
1824            self.state.seats.insert(seat.key.clone(), seat);
1825            let mut record = Judgement {
1826                judge: j + 1,
1827                seat: format!("judge-{}", j + 1),
1828                agent: agent_id,
1829                ranking: Vec::new(),
1830                reasons: BTreeMap::new(),
1831                confidence: None,
1832                order: orders[j].clone(),
1833                failed: None,
1834                duration_ms: 0,
1835            };
1836            match res {
1837                Ok((ranking, out)) => {
1838                    record.ranking = ranking.normalized();
1839                    record.reasons = ranking.reasons;
1840                    record.confidence = ranking.confidence;
1841                    record.duration_ms = out.duration_ms;
1842                    self.state.event(
1843                        "judge",
1844                        format!(
1845                            "judge {} ranked {}",
1846                            j + 1,
1847                            record.ranking.iter().collect::<String>()
1848                        ),
1849                    );
1850                }
1851                Err(e) => {
1852                    record.failed = Some(e.to_string());
1853                    self.state
1854                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1855                }
1856            }
1857            self.state.judgements.push(record);
1858            self.state.save()?;
1859        }
1860        Ok(())
1861    }
1862
1863    // ---------------------------------------------------------- deliberate
1864
1865    async fn deliberate(&mut self) -> Result<()> {
1866        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1867        // agent files with `magi task add` name the run that paid for it. The
1868        // prompt overlay is cloned alongside it because the waves borrow it
1869        // while `self` is mutably borrowed by the node's own bookkeeping.
1870        let run_id = self.state.id.clone();
1871        let prompts = self.state.config.prompts.clone();
1872        if !self.state.deliberation.is_empty() {
1873            return Ok(());
1874        }
1875        let tops: Vec<char> = self
1876            .state
1877            .judgements
1878            .iter()
1879            .filter_map(|j| j.ranking.first().copied())
1880            .collect();
1881        let rounds = self.state.config.graph.deliberate_rounds;
1882        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1883            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1884                self.state.event(
1885                    "deliberate",
1886                    format!("judges agreed on {} outright; no deliberation", tops[0]),
1887                );
1888            }
1889            self.state.status = RunStatus::Voting;
1890            self.state.save()?;
1891            return Ok(());
1892        }
1893
1894        self.state.status = RunStatus::Deliberating;
1895        self.state.event(
1896            "deliberate",
1897            format!(
1898                "split: first choices were {} — opening {rounds} round(s)",
1899                tops.iter().collect::<String>()
1900            ),
1901        );
1902
1903        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1904        let language = self.state.config.graph.language.clone();
1905        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1906        let sessions = self.state.config.graph.sessions;
1907        let artifacts = agent::artifacts_dir(&self.state.dir());
1908        let root = self.state.worktree_root();
1909        let base_short = short(&self.state.base_commit);
1910
1911        // Judges argue in sequence so that a turn can answer the one before it;
1912        // that is the difference between deliberation and three parallel
1913        // monologues.
1914        for round in 1..=rounds {
1915            let mut turns: Vec<DeliberationTurn> = Vec::new();
1916            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1917                if self.state.judgements[j].failed.is_some() {
1918                    continue;
1919                }
1920                let seat_key = format!("judge-{}", j + 1);
1921                let mut seat = self.seat(&seat_key, &spec.id);
1922                let transcript = self.transcript(&turns, j);
1923                let context = if has_context(&spec, &seat, sessions) {
1924                    None
1925                } else {
1926                    Some(self.candidate_block(&viable, &base_short))
1927                };
1928                let text = prompt::deliberate(
1929                    &self.state.instruction,
1930                    context.as_deref(),
1931                    &transcript,
1932                    round,
1933                    rounds,
1934                    &language,
1935                );
1936                let job = SeatJob {
1937                    spec,
1938                    seat: seat.clone(),
1939                    prompt: text,
1940                    cwd: root.join(format!("judge-{}", j + 1)),
1941                    timeout,
1942                    allow_write: false,
1943                    sessions,
1944                    artifacts: artifacts.clone(),
1945                    stem: format!("delib-{round}-judge-{}", j + 1),
1946                };
1947                let cache = self.state.config.cache_dir();
1948                let ctx = WaveCtx {
1949                    run: &run_id,
1950                    node: "deliberate",
1951                    prompts: &prompts,
1952                    cache: cache.as_deref(),
1953                };
1954                let (updated, out) =
1955                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1956                seat = updated;
1957                let agent_id = seat.agent.clone();
1958                let seat_key = seat.key.clone();
1959                self.state.seats.insert(seat.key.clone(), seat);
1960                let body = match out {
1961                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1962                    // Never read the CLI's raw error JSON as this judge's
1963                    // position — skip the seat instead, the same as any other
1964                    // failed turn.
1965                    AgentOutcome::Dropped(o) => {
1966                        let why =
1967                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1968                                "the CLI ended the stream without delivering its answer",
1969                            );
1970                        self.state.event(
1971                            "deliberate",
1972                            format!(
1973                                "judge {} skipped: the CLI dropped the stream ({why})",
1974                                j + 1
1975                            ),
1976                        );
1977                        continue;
1978                    }
1979                    AgentOutcome::Quota(o) => {
1980                        self.state.quota.push(QuotaLoss {
1981                            seat: seat_key,
1982                            node: "deliberate".to_owned(),
1983                            at: Timestamp::now(),
1984                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1985                        });
1986                        self.state.event(
1987                            "deliberate",
1988                            format!("judge {} skipped: rate limited (quota)", j + 1),
1989                        );
1990                        continue;
1991                    }
1992                    AgentOutcome::Failed(e) => {
1993                        self.state
1994                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1995                        continue;
1996                    }
1997                };
1998                let tentative = verdict::extract_json::<Position>(&body)
1999                    .ok()
2000                    .and_then(|p| p.tentative)
2001                    .and_then(|s| s.trim().chars().next())
2002                    .map(|c| c.to_ascii_uppercase());
2003                self.state.event(
2004                    "deliberate",
2005                    format!(
2006                        "round {round}: judge {} now favours {}",
2007                        j + 1,
2008                        tentative.map_or("—".to_owned(), |c| c.to_string())
2009                    ),
2010                );
2011                turns.push(DeliberationTurn {
2012                    judge: j + 1,
2013                    agent: agent_id,
2014                    body: blind::sanitize_prose(&body, &self.state.config.blind),
2015                    tentative,
2016                });
2017            }
2018            self.state
2019                .deliberation
2020                .push(DeliberationRound { round, turns });
2021            self.state.save()?;
2022        }
2023
2024        self.state.status = RunStatus::Voting;
2025        self.state.save()?;
2026        Ok(())
2027    }
2028
2029    // ---------------------------------------------------------------- vote
2030
2031    async fn vote(&mut self) -> Result<()> {
2032        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2033        // agent files with `magi task add` name the run that paid for it. The
2034        // prompt overlay is cloned alongside it because the waves borrow it
2035        // while `self` is mutably borrowed by the node's own bookkeeping.
2036        let run_id = self.state.id.clone();
2037        let prompts = self.state.config.prompts.clone();
2038        if !self.state.votes.is_empty() {
2039            return Ok(());
2040        }
2041        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2042        if viable.len() == 1 {
2043            return Ok(());
2044        }
2045        self.state.status = RunStatus::Voting;
2046
2047        let language = self.state.config.graph.language.clone();
2048        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2049        let sessions = self.state.config.graph.sessions;
2050        let artifacts = agent::artifacts_dir(&self.state.dir());
2051        let root = self.state.worktree_root();
2052        let base_short = short(&self.state.base_commit);
2053        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2054
2055        let mut jobs = Vec::new();
2056        let mut seats_at = Vec::new();
2057        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
2058            if self
2059                .state
2060                .judgements
2061                .get(j)
2062                .is_some_and(|r| r.failed.is_some())
2063            {
2064                continue;
2065            }
2066            let seat_key = format!("judge-{}", j + 1);
2067            let seat = self.seat(&seat_key, &spec.id);
2068            let mut text = prompt::final_vote(&viable, &language);
2069            if !has_context(&spec, &seat, sessions) {
2070                text = format!(
2071                    "{}\n\n# Candidates\n\n{}",
2072                    text,
2073                    self.candidate_block(&candidates, &base_short)
2074                );
2075            }
2076            jobs.push(SeatJob {
2077                spec,
2078                seat,
2079                prompt: text,
2080                cwd: root.join(format!("judge-{}", j + 1)),
2081                timeout,
2082                allow_write: false,
2083                sessions,
2084                artifacts: artifacts.clone(),
2085                stem: format!("vote-judge-{}", j + 1),
2086            });
2087            seats_at.push(j);
2088        }
2089
2090        self.state.event(
2091            "vote",
2092            format!(
2093                "collecting {} final votes one by one, privately",
2094                jobs.len()
2095            ),
2096        );
2097        let allowed = viable.clone();
2098        let mut quota_losses = Vec::new();
2099        let cache = self.state.config.cache_dir();
2100        let ctx = WaveCtx {
2101            run: &run_id,
2102            node: "vote",
2103            prompts: &prompts,
2104            cache: cache.as_deref(),
2105        };
2106        let results = ask_json_wave::<FinalVote>(
2107            jobs,
2108            Arc::clone(&self.sem),
2109            self.state.config.graph.retries,
2110            &ctx,
2111            &mut quota_losses,
2112            &mut self.state,
2113            &move |v: &FinalVote| match v.label() {
2114                Some(c) if allowed.contains(&c) => Ok(()),
2115                other => bail!("vote {other:?} is not one of {allowed:?}"),
2116            },
2117        )
2118        .await;
2119        self.state.quota.extend(quota_losses);
2120
2121        for (&j, (seat, res)) in seats_at.iter().zip(results) {
2122            let agent_id = seat.agent.clone();
2123            self.state.seats.insert(seat.key.clone(), seat);
2124            let initial = self
2125                .state
2126                .judgements
2127                .get(j)
2128                .and_then(|r| r.ranking.first().copied());
2129            let mut record = VoteRecord {
2130                judge: j + 1,
2131                agent: agent_id,
2132                vote: None,
2133                reason: String::new(),
2134                changed: false,
2135            };
2136            match res {
2137                Ok((v, _)) => {
2138                    record.vote = v.label();
2139                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2140                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
2141                    self.state.event(
2142                        "vote",
2143                        format!(
2144                            "judge {} voted {}{}",
2145                            j + 1,
2146                            record.vote.unwrap_or('?'),
2147                            if record.changed { " (changed)" } else { "" }
2148                        ),
2149                    );
2150                }
2151                Err(e) => {
2152                    self.state
2153                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
2154                }
2155            }
2156            self.state.votes.push(record);
2157            self.state.save()?;
2158        }
2159        Ok(())
2160    }
2161
2162    // --------------------------------------------------------------- tally
2163
2164    fn tally(&mut self) -> Result<()> {
2165        if self.state.tally.is_some() {
2166            return Ok(());
2167        }
2168        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
2169        let tops: Vec<char> = self
2170            .state
2171            .judgements
2172            .iter()
2173            .filter_map(|j| j.ranking.first().copied())
2174            .collect();
2175        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
2176
2177        // A judge whose private vote failed still counted once, in the initial
2178        // ranking; using it beats discarding a whole seat.
2179        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2180        let mut cast: Vec<char> = Vec::new();
2181        for (i, j) in self.state.judgements.iter().enumerate() {
2182            let vote = self
2183                .state
2184                .votes
2185                .iter()
2186                .find(|v| v.judge == i + 1)
2187                .and_then(|v| v.vote)
2188                .or_else(|| j.ranking.first().copied());
2189            if let Some(v) = vote {
2190                *first_choice.entry(v).or_insert(0) += 1;
2191                cast.push(v);
2192            }
2193        }
2194
2195        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
2196        for j in &self.state.judgements {
2197            let n = j.ranking.len();
2198            for (pos, label) in j.ranking.iter().enumerate() {
2199                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
2200            }
2201        }
2202
2203        let best = first_choice.values().copied().max().unwrap_or(0);
2204        let mut leaders: Vec<char> = first_choice
2205            .iter()
2206            .filter(|(_, v)| **v == best)
2207            .map(|(k, _)| *k)
2208            .collect();
2209        let mut tie_break = None;
2210        if leaders.len() > 1 {
2211            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
2212            let borda_leaders: Vec<char> = leaders
2213                .iter()
2214                .copied()
2215                .filter(|l| borda[l] == top_borda)
2216                .collect();
2217            tie_break = Some(if borda_leaders.len() == 1 {
2218                format!(
2219                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
2220                    leaders.len()
2221                )
2222            } else {
2223                format!(
2224                    "{} way tie on both first-choice votes and Borda points, broken by label order",
2225                    leaders.len()
2226                )
2227            });
2228            leaders = borda_leaders;
2229            leaders.sort_unstable();
2230        }
2231        let winner = *leaders
2232            .first()
2233            .or(viable.first())
2234            .context("no candidate to declare a winner from")?;
2235
2236        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
2237        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
2238        let deliberated = !self.state.deliberation.is_empty();
2239
2240        // Whose verdict is this? A rate-limited seat is absent even if it
2241        // ranked before the limit hit, so presence is measured against the
2242        // recorded losses, not just "did a ranking ever appear".
2243        let quota_seats: std::collections::BTreeSet<&str> =
2244            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2245        let mut present = 0usize;
2246        for (i, j) in self.state.judgements.iter().enumerate() {
2247            if quota_seats.contains(j.seat.as_str()) {
2248                continue;
2249            }
2250            let ranked = !j.ranking.is_empty() && j.failed.is_none();
2251            let voted = self
2252                .state
2253                .votes
2254                .iter()
2255                .any(|v| v.judge == i + 1 && v.vote.is_some());
2256            if ranked || voted {
2257                present += 1;
2258            }
2259        }
2260        // Strict majority of the configured panel. A bare majority is real
2261        // signal we can act on, while a minority verdict must never stand in
2262        // for a healthy one. A one-candidate run needs no panel at all, and
2263        // `judges` stays `0` rather than the roster size a panel that never
2264        // sat would otherwise be credited with.
2265        let needs_quorum = viable.len() > 1;
2266        let judges_total = if needs_quorum {
2267            self.roles.judges.len()
2268        } else {
2269            0
2270        };
2271        let quorum = if needs_quorum {
2272            judges_total / 2 + 1
2273        } else {
2274            0
2275        };
2276        let met_quorum = !needs_quorum || present >= quorum;
2277        let uncontested = (!needs_quorum).then(|| {
2278            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
2279        });
2280
2281        self.state.event(
2282            "tally",
2283            match &uncontested {
2284                Some(reason) => format!("winner {winner} — {reason}"),
2285                None => format!(
2286                    "winner {winner} — votes {} | initial {} | {} changed | \
2287                     {present}/{judges_total} judges{}",
2288                    first_choice
2289                        .iter()
2290                        .map(|(k, v)| format!("{k}:{v}"))
2291                        .collect::<Vec<_>>()
2292                        .join(" "),
2293                    if unanimous_initial {
2294                        "unanimous"
2295                    } else {
2296                        "split"
2297                    },
2298                    changed_votes,
2299                    if met_quorum {
2300                        String::new()
2301                    } else {
2302                        format!(" — below quorum ({quorum} required)")
2303                    },
2304                ),
2305            },
2306        );
2307        if !met_quorum {
2308            self.state.event(
2309                "stall",
2310                format!(
2311                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
2312                     the run stops here, resumable"
2313                ),
2314            );
2315        }
2316        self.state.tally = Some(Tally {
2317            first_choice,
2318            borda,
2319            winner,
2320            rankings: tops.len(),
2321            unanimous_initial,
2322            deliberated,
2323            changed_votes,
2324            unanimous_final,
2325            tie_break,
2326            judges: judges_total,
2327            present,
2328            quorum,
2329            met_quorum,
2330            uncontested,
2331        });
2332        self.state.status = if met_quorum {
2333            RunStatus::Reviewing
2334        } else {
2335            RunStatus::Stalled
2336        };
2337        self.state.save()?;
2338        Ok(())
2339    }
2340
2341    // ------------------------------------------------------------- recover
2342
2343    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
2344    /// resumed toward completion once the transient cause clears.
2345    ///
2346    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
2347    /// it toward the quorum, which is exactly the set of seats whose absence
2348    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
2349    /// not depend on which node happened to hit the limit), or an ordinary
2350    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
2351    /// seat is never disturbed.
2352    ///
2353    /// A seat that now answers with a usable ranking is "recovered": its
2354    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
2355    /// `tally` counts it present again), and its vote re-collected. A seat that
2356    /// still fails keeps its loss and stays absent.
2357    ///
2358    /// Returns `true` when the re-tally restores the quorum (the run may proceed
2359    /// to review/gate/merge), `false` when it is still below quorum (the run
2360    /// stays `Stalled`, still resumable for a later retry).
2361    #[allow(clippy::too_many_lines)]
2362    async fn recover_stall(&mut self) -> Result<bool> {
2363        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2364        // agent files with `magi task add` name the run that paid for it. The
2365        // prompt overlay is cloned alongside it because the waves borrow it
2366        // while `self` is mutably borrowed by the node's own bookkeeping.
2367        let run_id = self.state.id.clone();
2368        let prompts = self.state.config.prompts.clone();
2369        // Absent seats = quota-lost at any node, or failed outright. Mirroring
2370        // `tally`'s presence test (rather than the old quota-judge/vote filter)
2371        // is what keeps a non-quota collapse — or a quota loss recorded at the
2372        // deliberate node — from being a permanent dead-end on `--resume`.
2373        let quota_seats: BTreeSet<&str> =
2374            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
2375        let absent: Vec<String> = self
2376            .state
2377            .judgements
2378            .iter()
2379            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2380            .map(|j| j.seat.clone())
2381            .collect();
2382        if absent.is_empty() {
2383            return Ok(false);
2384        }
2385        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2386        if viable.len() <= 1 {
2387            return Ok(false);
2388        }
2389        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2390        let language = self.state.config.graph.language.clone();
2391        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2392        let sessions = self.state.config.graph.sessions;
2393        let artifacts = agent::artifacts_dir(&self.state.dir());
2394        let root = self.state.worktree_root();
2395        let base_short = short(&self.state.base_commit);
2396        let candidates: Vec<Candidate> = viable.clone();
2397
2398        // Map each absent seat key to its 0-based position in `roles.judges`.
2399        let mut positions: Vec<usize> = absent
2400            .iter()
2401            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2402            .collect();
2403        if positions.is_empty() {
2404            return Ok(false);
2405        }
2406        positions.sort_unstable();
2407        positions.dedup();
2408
2409        // Re-rank the lost seats, one blind prompt each.
2410        let mut judge_jobs = Vec::new();
2411        for &j in &positions {
2412            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2413            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2414            let seat_key = format!("judge-{}", j + 1);
2415            let spec = self.roles.judges[j].clone();
2416            let seat = self.seat(&seat_key, &spec.id);
2417            judge_jobs.push(SeatJob {
2418                spec,
2419                seat,
2420                prompt: prompt::judge(
2421                    &self.state.instruction,
2422                    &views,
2423                    self.roles.judges.len(),
2424                    &base_short,
2425                    &language,
2426                ),
2427                cwd: root.join(seat_key),
2428                timeout,
2429                allow_write: false,
2430                sessions,
2431                artifacts: artifacts.clone(),
2432                stem: format!("judge-{}-recover", j + 1),
2433            });
2434        }
2435
2436        let labels_for_check = labels.clone();
2437        let mut judge_losses = Vec::new();
2438        let retries = self.state.config.graph.retries;
2439        let cache = self.state.config.cache_dir();
2440        let ctx = WaveCtx {
2441            run: &run_id,
2442            node: "judge",
2443            prompts: &prompts,
2444            cache: cache.as_deref(),
2445        };
2446        let results = ask_json_wave::<Ranking>(
2447            judge_jobs,
2448            Arc::clone(&self.sem),
2449            retries,
2450            &ctx,
2451            &mut judge_losses,
2452            &mut self.state,
2453            &move |r: &Ranking| r.validate(&labels_for_check),
2454        )
2455        .await;
2456
2457        // Refresh the judgement of every seat that ranked again.
2458        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2459        for (&j, (seat, res)) in positions.iter().zip(results) {
2460            self.state.seats.insert(seat.key.clone(), seat);
2461            let record = &mut self.state.judgements[j];
2462            match res {
2463                Ok((ranking, out)) => {
2464                    record.ranking = ranking.normalized();
2465                    record.reasons = ranking.reasons;
2466                    record.confidence = ranking.confidence;
2467                    record.failed = None;
2468                    record.duration_ms = out.duration_ms;
2469                    recovered.insert(j);
2470                    self.state.event(
2471                        "recover",
2472                        format!("judge {} ranked again after the limit", j + 1),
2473                    );
2474                }
2475                Err(e) => {
2476                    self.state
2477                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2478                }
2479            }
2480        }
2481
2482        // Re-ask the votes of the seats that recovered a ranking.
2483        let mut vote_jobs = Vec::new();
2484        let mut vote_pos: Vec<usize> = Vec::new();
2485        for &j in &recovered {
2486            let seat_key = format!("judge-{}", j + 1);
2487            let spec = self.roles.judges[j].clone();
2488            let seat = self.seat(&seat_key, &spec.id);
2489            let mut text = prompt::final_vote(&labels, &language);
2490            if !has_context(&spec, &seat, sessions) {
2491                text = format!(
2492                    "{}\n\n# Candidates\n\n{}",
2493                    text,
2494                    self.candidate_block(&candidates, &base_short)
2495                );
2496            }
2497            vote_jobs.push(SeatJob {
2498                spec,
2499                seat,
2500                prompt: text,
2501                cwd: root.join(seat_key),
2502                timeout,
2503                allow_write: false,
2504                sessions,
2505                artifacts: artifacts.clone(),
2506                stem: format!("vote-judge-{}-recover", j + 1),
2507            });
2508            vote_pos.push(j);
2509        }
2510        let allowed = labels.clone();
2511        let mut vote_losses = Vec::new();
2512        let vote_retries = self.state.config.graph.retries;
2513        let vote_cache = self.state.config.cache_dir();
2514        let ctx = WaveCtx {
2515            run: &run_id,
2516            node: "vote",
2517            prompts: &prompts,
2518            cache: vote_cache.as_deref(),
2519        };
2520        let votes = ask_json_wave::<FinalVote>(
2521            vote_jobs,
2522            Arc::clone(&self.sem),
2523            vote_retries,
2524            &ctx,
2525            &mut vote_losses,
2526            &mut self.state,
2527            &move |v: &FinalVote| match v.label() {
2528                Some(c) if allowed.contains(&c) => Ok(()),
2529                other => bail!("vote {other:?} is not one of {allowed:?}"),
2530            },
2531        )
2532        .await;
2533        for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
2534            let agent_id = seat.agent.clone();
2535            self.state.seats.insert(seat.key.clone(), seat);
2536            match res {
2537                Ok((v, _)) => {
2538                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2539                        rec.vote = v.label();
2540                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2541                    } else {
2542                        self.state.votes.push(VoteRecord {
2543                            judge: j + 1,
2544                            agent: agent_id,
2545                            vote: v.label(),
2546                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2547                            changed: false,
2548                        });
2549                    }
2550                    self.state.event(
2551                        "recover",
2552                        format!("judge {} voted again after the limit", j + 1),
2553                    );
2554                }
2555                Err(e) => {
2556                    self.state
2557                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2558                }
2559            }
2560        }
2561
2562        // A seat that ranked again is present even if its re-vote failed —
2563        // `tally` falls back to the initial ranking's first choice — so clear
2564        // its quota loss. Seats that still fail keep theirs and stay absent.
2565        if !recovered.is_empty() {
2566            let recovered_keys: BTreeSet<String> = recovered
2567                .iter()
2568                .map(|&j| format!("judge-{}", j + 1))
2569                .collect();
2570            self.state
2571                .quota
2572                .retain(|q| !recovered_keys.contains(&q.seat));
2573        }
2574
2575        // Recompute the verdict from the refreshed panel.
2576        self.state.tally = None;
2577        self.tally()?;
2578        Ok(self
2579            .state
2580            .tally
2581            .as_ref()
2582            .map(|t| t.met_quorum)
2583            .unwrap_or(false))
2584    }
2585
2586    // ----------------------------------------------------------------- fold
2587
2588    async fn fold_losers(&mut self) -> Result<()> {
2589        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
2590            return Ok(());
2591        };
2592        let repo = self.state.repo.clone();
2593        let mut folded = Vec::new();
2594        for i in 0..self.state.candidates.len() {
2595            let c = &self.state.candidates[i];
2596            if c.label == winner || c.folded {
2597                continue;
2598            }
2599            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
2600            git::worktree_remove(&repo, &wt).await.ok();
2601            git::branch_delete(&repo, &branch).await.ok();
2602            self.state.candidates[i].folded = true;
2603            folded.push(label.to_string());
2604        }
2605        // The judges are finished; their checkouts are pure cost from here.
2606        let root = self.state.worktree_root();
2607        for j in 1..=self.roles.judges.len() {
2608            let wt = root.join(format!("judge-{j}"));
2609            if wt.exists() {
2610                git::worktree_remove(&repo, &wt).await.ok();
2611            }
2612        }
2613        // The design-deliberation stage is finished by the time a tally
2614        // exists — same reasoning as the judges above.
2615        if self.state.config.graph.advise {
2616            for k in 1..=self.state.config.graph.advisors {
2617                let wt = root.join(format!("advisor-{k}"));
2618                if wt.exists() {
2619                    git::worktree_remove(&repo, &wt).await.ok();
2620                }
2621            }
2622        }
2623        if !folded.is_empty() {
2624            self.state
2625                .event("fold", format!("folded candidates {}", folded.join(", ")));
2626            self.state.save()?;
2627        }
2628        Ok(())
2629    }
2630
2631    // ------------------------------------------------------------ base sync
2632
2633    /// Land the winner's tree on the current tip of `<remote>/<base>` before
2634    /// anything verifies it.
2635    ///
2636    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
2637    /// read whatever is checked out in the winner's worktree. Left alone that
2638    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
2639    /// it when the run *branched* - and a run takes long enough that the base
2640    /// has usually moved by the time it gets here. A gate that ran there
2641    /// answers "green on the commit this run started from", not "green on
2642    /// what is about to land", and the difference showed up three times in
2643    /// one day as a green run whose merge would have reverted a file another
2644    /// pull request had already landed.
2645    ///
2646    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
2647    /// implementation of the same idea: `land::Step::Rebase` already worked
2648    /// out the rules - throwaway worktree, conflict stops and reports rather
2649    /// than feeding a fixer, nothing runs in the primary tree - and a second
2650    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
2651    /// warns about ("two answers to a question nobody notices until a diff is
2652    /// wrong").
2653    ///
2654    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
2655    /// so it survives a park/resume. A conflict or a push failure sets
2656    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
2657    /// as they were - untouched, for a person to look at - which is also what
2658    /// makes re-entering this function afterwards a no-op instead of a second
2659    /// attempt at the same wall.
2660    async fn sync_to_base(&mut self) -> Result<()> {
2661        if self
2662            .state
2663            .base_sync
2664            .as_ref()
2665            .is_some_and(|s| s.conflict.is_some())
2666        {
2667            return Ok(());
2668        }
2669        let Some(winner) = self.state.winner().cloned() else {
2670            return Ok(());
2671        };
2672
2673        let repo = self.state.repo.clone();
2674        let remote = self.state.config.merge.remote.clone();
2675        let base_branch = self.state.base_branch.clone();
2676        let tracking = format!("{remote}/{base_branch}");
2677
2678        git::fetch(&repo, &remote, &base_branch).await.ok();
2679        // No network, or the remote never had this branch: `resolve_base`
2680        // already treats that as non-fatal at branch time, and a run that got
2681        // this far must not be blocked by it here either.
2682        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
2683            return Ok(());
2684        };
2685
2686        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2687        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
2688        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
2689
2690        if behind == 0 {
2691            self.state.base_sync = Some(BaseSync {
2692                tip,
2693                behind: 0,
2694                attempts,
2695                conflict: None,
2696            });
2697            self.state.save()?;
2698            return Ok(());
2699        }
2700
2701        if attempts >= BASE_SYNC_ROUNDS {
2702            let why = format!(
2703                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
2704                 rebase(s); rebasing again would only race it",
2705                winner.branch
2706            );
2707            self.state.status = RunStatus::Blocked;
2708            self.state.base_sync = Some(BaseSync {
2709                tip,
2710                behind,
2711                attempts,
2712                conflict: Some(why.clone()),
2713            });
2714            self.state.event("land", why);
2715            self.state.save()?;
2716            return Ok(());
2717        }
2718
2719        self.state.event(
2720            "land",
2721            format!(
2722                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
2723                winner.branch
2724            ),
2725        );
2726        self.state.save()?;
2727
2728        let scratch = self.state.dir().join("base-sync");
2729        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
2730        let attempts = attempts + 1;
2731        match rebased {
2732            Ok(None) => {
2733                // The branch ref moved, but a worktree that already had it
2734                // checked out (the winner's) was not told; sync its index and
2735                // files before anything reads them.
2736                git::sync_to_head(&winner.worktree).await?;
2737                self.state.base_sync = Some(BaseSync {
2738                    tip: tip.clone(),
2739                    behind: 0,
2740                    attempts,
2741                    conflict: None,
2742                });
2743                self.state
2744                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
2745            }
2746            Ok(Some(conflict)) => {
2747                let why = format!(
2748                    "{} conflicts with {tracking} and did not rebase: {}",
2749                    winner.branch,
2750                    conflict.chars().take(600).collect::<String>()
2751                );
2752                self.state.status = RunStatus::Blocked;
2753                self.state.base_sync = Some(BaseSync {
2754                    tip,
2755                    behind,
2756                    attempts,
2757                    conflict: Some(why.clone()),
2758                });
2759                self.state.event("land", why);
2760            }
2761            Err(e) => {
2762                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
2763                self.state.status = RunStatus::Blocked;
2764                self.state.base_sync = Some(BaseSync {
2765                    tip,
2766                    behind,
2767                    attempts,
2768                    conflict: Some(why.clone()),
2769                });
2770                self.state.event("land", why);
2771            }
2772        }
2773        self.state.save()?;
2774        Ok(())
2775    }
2776
2777    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
2778    /// last landed the winner on, once it has run, else the commit the run
2779    /// branched from.
2780    ///
2781    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
2782    /// and `vote` all happen before there is a winner to rebase, so they
2783    /// compare every candidate against the branch point on purpose, and a
2784    /// base that moves after they are already done cannot change an answer
2785    /// they already gave.
2786    fn landing_base(&self) -> String {
2787        self.state
2788            .base_sync
2789            .as_ref()
2790            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2791    }
2792
2793    // --------------------------------------------------------------- review
2794
2795    async fn review_loop(&mut self) -> Result<()> {
2796        // A base that would not rebase is a person's decision, not a review
2797        // round: nothing here would change the answer, and reviewers and a
2798        // fixer would be spending real budget on a tree that cannot land
2799        // regardless of what they find.
2800        if self
2801            .state
2802            .base_sync
2803            .as_ref()
2804            .is_some_and(|s| s.conflict.is_some())
2805        {
2806            return Ok(());
2807        }
2808        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2809        // agent files with `magi task add` name the run that paid for it. The
2810        // prompt overlay is cloned alongside it because the waves borrow it
2811        // while `self` is mutably borrowed by the node's own bookkeeping.
2812        let run_id = self.state.id.clone();
2813        let prompts = self.state.config.prompts.clone();
2814        let Some(winner) = self.state.winner().cloned() else {
2815            return Ok(());
2816        };
2817        let max_rounds = self.state.config.graph.review_rounds;
2818        // A clean round, an exhausted round budget, or a stalled tree (see
2819        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
2820        // they are recorded — recomputed here, not read off `status`, so a
2821        // reentry into a run that already stopped restates the identical
2822        // verdict instead of silently handing back whatever an earlier node
2823        // in this same walk clobbered `status` to (a solo-candidate
2824        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
2825        // below runs an empty range once the budget is spent, and would
2826        // otherwise fall through without touching `status` at all.
2827        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
2828            self.state.status = status;
2829            self.state.save()?;
2830            return Ok(());
2831        }
2832        self.state.status = RunStatus::Reviewing;
2833
2834        let repo = self.state.repo.clone();
2835        let root = self.state.worktree_root();
2836        let language = self.state.config.graph.language.clone();
2837        let sessions = self.state.config.graph.sessions;
2838        let artifacts = agent::artifacts_dir(&self.state.dir());
2839        let base = self.landing_base();
2840        let base_short = short(&base);
2841        let reviewers = self.roles.reviewers.clone();
2842        let shell = self.state.config.shell();
2843
2844        let mut prev_e2e: Option<String> = None;
2845        for round in (self.state.reviews.len() + 1)..=max_rounds {
2846            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2847            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2848            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2849
2850            // Each reviewer gets its own detached checkout of exactly this
2851            // commit: nobody can perturb the winner's tree, and the fixer can
2852            // keep working without racing a reviewer.
2853            let mut jobs = Vec::new();
2854            for (r, spec) in reviewers.iter().cloned().enumerate() {
2855                let wt = root.join(format!("review-{}", r + 1));
2856                if wt.exists() {
2857                    git::reset_detached(&wt, &head).await?;
2858                } else {
2859                    git::worktree_add_detached(&repo, &wt, &head).await?;
2860                }
2861                let seat_key = format!("review-{}", r + 1);
2862                let seat = self.seat(&seat_key, &spec.id);
2863                jobs.push(SeatJob {
2864                    prompt: prompt::review(&prompt::ReviewCtx {
2865                        instruction: &self.state.instruction,
2866                        branch: &winner.branch,
2867                        base_short: &base_short,
2868                        stat: &stat,
2869                        patch: &patch,
2870                        e2e: prev_e2e.as_deref(),
2871                        reviewers: reviewers.len(),
2872                        round,
2873                        rounds: max_rounds,
2874                        // A review-only run has no rankings, so nothing
2875                        // competed for this patch and the reviewer is told so.
2876                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2877                        lens: Lens::for_seat(r),
2878                        language: &language,
2879                    }),
2880                    spec,
2881                    seat,
2882                    cwd: wt,
2883                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2884                    allow_write: false,
2885                    sessions,
2886                    artifacts: artifacts.clone(),
2887                    stem: format!("review-{round}-{}", r + 1),
2888                });
2889            }
2890
2891            self.state.event(
2892                "review",
2893                format!(
2894                    "round {round}: {} reviewers on {}",
2895                    jobs.len(),
2896                    short(&head)
2897                ),
2898            );
2899            let mut quota_losses = Vec::new();
2900            let review_retries = self.state.config.graph.retries;
2901            let review_cache = self.state.config.cache_dir();
2902            let ctx = WaveCtx {
2903                run: &run_id,
2904                node: "review",
2905                prompts: &prompts,
2906                cache: review_cache.as_deref(),
2907            };
2908            let results = ask_json_wave::<Review>(
2909                jobs,
2910                Arc::clone(&self.sem),
2911                review_retries,
2912                &ctx,
2913                &mut quota_losses,
2914                &mut self.state,
2915                &|_: &Review| Ok(()),
2916            )
2917            .await;
2918            // Counted before the move below: how many of *this* round's
2919            // reviewer seats were lost to their own rate limit, as opposed to
2920            // a crash, a timeout, or unparsable output — see `round_is_clean`.
2921            let round_quota_missing = quota_losses.len();
2922            self.state.quota.extend(quota_losses);
2923
2924            let mut records = Vec::new();
2925            let mut all_findings = Vec::new();
2926            for (r, (seat, res)) in results.into_iter().enumerate() {
2927                let agent_id = seat.agent.clone();
2928                self.state.seats.insert(seat.key.clone(), seat);
2929                let mut record = ReviewRecord {
2930                    reviewer: r + 1,
2931                    agent: agent_id,
2932                    summary: String::new(),
2933                    findings: Vec::new(),
2934                    vote: None,
2935                    failed: None,
2936                    duration_ms: 0,
2937                };
2938                match res {
2939                    Ok((review, out)) => {
2940                        // Sanitized here, at the point every other piece of
2941                        // agent prose in this file is (candidate summaries,
2942                        // deliberation turns, vote reasons): a reviewer's own
2943                        // words are the one thing about it that could name
2944                        // it, and reconsideration below broadcasts this same
2945                        // summary and these same findings to every other
2946                        // seat on the panel.
2947                        record.summary =
2948                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
2949                        record.vote = Some(review.vote);
2950                        record.duration_ms = out.duration_ms;
2951                        for (n, mut f) in review.findings.into_iter().enumerate() {
2952                            // ids are magi's, never the agent's: the fixer's
2953                            // adoption report is keyed by them.
2954                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2955                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
2956                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
2957                            // `file` is agent-supplied prose too, never
2958                            // checked against the real tree — the same
2959                            // exposure `title`/`detail` above have, just in
2960                            // a field easy to forget because it looks like a
2961                            // path rather than free text.
2962                            f.file = f
2963                                .file
2964                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
2965                            all_findings.push(f.clone());
2966                            record.findings.push(f);
2967                        }
2968                        self.state.event(
2969                            "review",
2970                            format!(
2971                                "round {round}: reviewer {} voted {} with {} finding(s)",
2972                                r + 1,
2973                                review.vote.label(),
2974                                record.findings.len()
2975                            ),
2976                        );
2977                    }
2978                    Err(e) => {
2979                        record.failed = Some(e.to_string());
2980                        self.state.event(
2981                            "review",
2982                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2983                        );
2984                    }
2985                }
2986                records.push(record);
2987            }
2988
2989            // Tally the round's votes and, if they split, spend the one
2990            // round of reconsideration the split -> deliberate -> revote
2991            // shape `judge`/`vote` use for the panel, sized down to what a
2992            // read-only review round can afford: one round, and a revote
2993            // rather than an argument, because the panel already wrote its
2994            // reasoning down as findings the first time around.
2995            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
2996            let vote_split =
2997                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
2998            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
2999            if vote_split {
3000                self.state.event(
3001                    "review",
3002                    format!(
3003                        "round {round}: votes split ({}) — one round of reconsideration",
3004                        initial_votes
3005                            .iter()
3006                            .map(|v| v.label())
3007                            .collect::<Vec<_>>()
3008                            .join(", ")
3009                    ),
3010                );
3011                // Seats read every seat's findings and votes, still numbered
3012                // and never named — the same anonymity `review` itself keeps.
3013                let panel: Vec<ReviewSeatReport<'_>> = records
3014                    .iter()
3015                    .filter_map(|r| {
3016                        r.vote.map(|vote| ReviewSeatReport {
3017                            reviewer: r.reviewer,
3018                            vote,
3019                            summary: &r.summary,
3020                            findings: &r.findings,
3021                        })
3022                    })
3023                    .collect();
3024
3025                let mut jobs = Vec::new();
3026                let mut seats_at = Vec::new();
3027                for (r, spec) in reviewers.iter().cloned().enumerate() {
3028                    // A seat with no initial vote has nothing to reconsider
3029                    // from and stays absent, the same as it stayed absent
3030                    // from `panel` above.
3031                    if records[r].vote.is_none() {
3032                        continue;
3033                    }
3034                    let wt = root.join(format!("review-{}", r + 1));
3035                    let seat_key = format!("review-{}", r + 1);
3036                    let seat = self.seat(&seat_key, &spec.id);
3037                    // A seat with no live session has already forgotten the
3038                    // initial review's prompt — restate the patch it is
3039                    // voting on, the same as `deliberate`/`vote` do for a
3040                    // judge in the same position.
3041                    let patch_ctx = if has_context(&spec, &seat, sessions) {
3042                        None
3043                    } else {
3044                        Some(ReviewPatch {
3045                            branch: &winner.branch,
3046                            base_short: &base_short,
3047                            stat: &stat,
3048                            patch: &patch,
3049                        })
3050                    };
3051                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
3052                        instruction: &self.state.instruction,
3053                        reviewer: r + 1,
3054                        lens: Lens::for_seat(r),
3055                        panel: &panel,
3056                        patch: patch_ctx,
3057                        round,
3058                        rounds: max_rounds,
3059                        language: &language,
3060                    });
3061                    jobs.push(SeatJob {
3062                        prompt,
3063                        spec,
3064                        seat,
3065                        cwd: wt,
3066                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
3067                        allow_write: false,
3068                        sessions,
3069                        artifacts: artifacts.clone(),
3070                        stem: format!("review-{round}-reconsider-{}", r + 1),
3071                    });
3072                    seats_at.push(r);
3073                }
3074
3075                let mut recon_quota_losses = Vec::new();
3076                let recon_cache = self.state.config.cache_dir();
3077                let recon_ctx = WaveCtx {
3078                    run: &run_id,
3079                    node: "review",
3080                    prompts: &prompts,
3081                    cache: recon_cache.as_deref(),
3082                };
3083                let recon_results = ask_json_wave::<ReviewRevote>(
3084                    jobs,
3085                    Arc::clone(&self.sem),
3086                    review_retries,
3087                    &recon_ctx,
3088                    &mut recon_quota_losses,
3089                    &mut self.state,
3090                    &|_: &ReviewRevote| Ok(()),
3091                )
3092                .await;
3093                self.state.quota.extend(recon_quota_losses);
3094
3095                for (&r, (seat, res)) in seats_at.iter().zip(recon_results) {
3096                    let agent_id = seat.agent.clone();
3097                    self.state.seats.insert(seat.key.clone(), seat);
3098                    let mut rec = ReviewRevoteRecord {
3099                        reviewer: r + 1,
3100                        agent: agent_id,
3101                        vote: None,
3102                        reason: String::new(),
3103                        failed: None,
3104                    };
3105                    match res {
3106                        Ok((rv, _)) => {
3107                            rec.vote = Some(rv.vote);
3108                            rec.reason =
3109                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
3110                            self.state.event(
3111                                "review",
3112                                format!(
3113                                    "round {round}: reviewer {} revoted {}",
3114                                    r + 1,
3115                                    rv.vote.label()
3116                                ),
3117                            );
3118                        }
3119                        Err(e) => {
3120                            rec.failed = Some(e.to_string());
3121                            self.state.event(
3122                                "review",
3123                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
3124                            );
3125                        }
3126                    }
3127                    reconsideration.push(rec);
3128                }
3129            } else if initial_votes.len() > 1 {
3130                self.state.event(
3131                    "review",
3132                    format!(
3133                        "round {round}: votes agreed ({}) — no reconsideration",
3134                        initial_votes[0].label()
3135                    ),
3136                );
3137            }
3138
3139            // The final vote per seat is its revote where reconsideration
3140            // ran and answered, its initial vote otherwise — the same
3141            // fallback `tally` uses for a judge whose private vote failed.
3142            let final_votes: Vec<ReviewVote> = records
3143                .iter()
3144                .filter_map(|r| {
3145                    reconsideration
3146                        .iter()
3147                        .find(|rv| rv.reviewer == r.reviewer)
3148                        .and_then(|rv| rv.vote)
3149                        .or(r.vote)
3150                })
3151                .collect();
3152            let round_verdict = ReviewVote::worst(final_votes);
3153
3154            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
3155            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3156            // A round that already has a blocking finding and a round left to
3157            // try is going back to the fixer no matter what `verify.e2e`
3158            // says, so running it first only spends the loop's slowest step
3159            // (minutes, for a Rust repo's full test suite) on a head about
3160            // to be rewritten. Deferred, never skipped: `verify.e2e` still
3161            // runs once a round has no blocking findings left (see
3162            // `round_is_clean`, which a deferred — empty — `e2e` can never
3163            // satisfy since `blocking` is nonzero whenever this branch is
3164            // taken), and `stop_reviewing` forces a real run before it will
3165            // ever read a deferred round as green.
3166            let defer_e2e =
3167                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
3168            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
3169                let reason =
3170                    format!("{blocking} blocking finding(s) already required a fix this round");
3171                self.state.event(
3172                    "verify",
3173                    format!(
3174                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
3175                         {}); it will run once a round has none left",
3176                        short(&head)
3177                    ),
3178                );
3179                (Vec::new(), false, true, Some(reason))
3180            } else {
3181                let e2e_commands = self.state.config.verify.e2e.clone();
3182                let cache_dir = self.state.config.cache_dir();
3183                let context = format!("round {round}");
3184                let (e2e, verify_retried) = with_cache_lease(
3185                    &mut self.state,
3186                    cache_dir.as_deref(),
3187                    "e2e",
3188                    "e2e",
3189                    &winner.worktree,
3190                    &head,
3191                    verify_timeout,
3192                    &context,
3193                    |state, budget| {
3194                        let shell = shell.clone();
3195                        let e2e_commands = e2e_commands.clone();
3196                        let worktree = winner.worktree.clone();
3197                        let context = context.clone();
3198                        async move {
3199                            run_e2e_with_retry(
3200                                state,
3201                                &shell,
3202                                &e2e_commands,
3203                                &worktree,
3204                                budget,
3205                                &context,
3206                            )
3207                            .await
3208                        }
3209                    },
3210                )
3211                .await;
3212                (e2e, verify_retried, false, None)
3213            };
3214
3215            let e2e_failures: String = e2e
3216                .iter()
3217                .filter(|o| !o.ok())
3218                .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
3219                .collect();
3220
3221            let expected = records.len();
3222            let answered = records.iter().filter(|r| r.failed.is_none()).count();
3223            let incomplete = answered < expected;
3224            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
3225            let policy = self.state.config.graph.incomplete_review;
3226            let clean = round_is_clean(
3227                blocking,
3228                e2e_ok,
3229                answered,
3230                expected,
3231                round_quota_missing,
3232                policy,
3233            );
3234
3235            let mut round_record = ReviewRound {
3236                round,
3237                head: head.clone(),
3238                verified_head: None,
3239                reviews: records,
3240                e2e,
3241                verify_retried,
3242                e2e_deferred,
3243                e2e_defer_reason,
3244                fix: None,
3245                blocking,
3246                answered,
3247                expected,
3248                clean,
3249                progressed: false,
3250                vote_split,
3251                reconsideration,
3252                verdict: round_verdict,
3253            };
3254
3255            if incomplete {
3256                let missing: Vec<String> = round_record
3257                    .reviews
3258                    .iter()
3259                    .filter(|r| r.failed.is_some())
3260                    .map(|r| format!("review-{}", r.reviewer))
3261                    .collect();
3262                self.state.event(
3263                    "review",
3264                    format!(
3265                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
3266                        missing.join(", ")
3267                    ),
3268                );
3269            }
3270
3271            if clean {
3272                self.state.event(
3273                    "review",
3274                    if incomplete && policy == IncompleteReviewPolicy::Warn {
3275                        format!(
3276                            "round {round}: clean (warn policy, incomplete panel) — no \
3277                             blocking findings from the seats that answered, verification green"
3278                        )
3279                    } else if incomplete {
3280                        format!(
3281                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
3282                             quorum) — no blocking findings from the seats that answered, \
3283                             verification green",
3284                            expected - answered
3285                        )
3286                    } else {
3287                        format!("round {round}: clean — no blocking findings, verification green")
3288                    },
3289                );
3290                self.state.reviews.push(round_record);
3291                self.state.status = RunStatus::Gating;
3292                self.state.save()?;
3293                return Ok(());
3294            }
3295
3296            // Nothing was raised and verification passed, but not every seat
3297            // answered and `round_is_clean` still refused to call it clean —
3298            // either a seat is missing for a reason other than its own quota
3299            // (a crash, a timeout, unparsable output — worth another try), or
3300            // every seat that could have answered lost its quota and nobody
3301            // is left to decide on: re-review rather than send the fixer
3302            // after a round with nothing to fix.
3303            if incomplete && blocking == 0 && e2e_ok {
3304                self.state.reviews.push(round_record);
3305                self.state.save()?;
3306                if round == max_rounds {
3307                    self.state.status = RunStatus::Blocked;
3308                    self.state.event(
3309                        "review",
3310                        format!(
3311                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
3312                             refusing to call it clean",
3313                            expected - answered
3314                        ),
3315                    );
3316                    return Ok(());
3317                }
3318                prev_e2e = None;
3319                continue;
3320            }
3321
3322            if round == max_rounds {
3323                self.state.reviews.push(round_record);
3324                return self
3325                    .stop_reviewing(
3326                        &format!(
3327                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
3328                        ),
3329                        &shell,
3330                        &winner.worktree,
3331                    )
3332                    .await;
3333            }
3334
3335            // Fix. The winner's own implementer seat continues its conversation:
3336            // the competition is over, so context is pure benefit now.
3337            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
3338                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
3339                _ => (
3340                    self.state
3341                        .config
3342                        .agent(&winner.agent)
3343                        .cloned()
3344                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
3345                    format!("impl-{}", winner.label),
3346                ),
3347            };
3348            let seat = self.seat(&fix_seat_key, &fix_spec.id);
3349            let blocking_findings: Vec<_> = all_findings
3350                .iter()
3351                .filter(|f| f.severity.blocks())
3352                .cloned()
3353                .collect();
3354            let job = SeatJob {
3355                prompt: prompt::fix(
3356                    &self.state.instruction,
3357                    &blocking_findings,
3358                    (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
3359                    e2e_deferred,
3360                    round,
3361                    max_rounds,
3362                    &language,
3363                ),
3364                spec: fix_spec.clone(),
3365                seat,
3366                cwd: winner.worktree.clone(),
3367                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
3368                allow_write: true,
3369                sessions,
3370                artifacts: artifacts.clone(),
3371                stem: format!("fix-{round}"),
3372            };
3373            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
3374            let cache = self.state.config.cache_dir();
3375            let ctx = WaveCtx {
3376                run: &run_id,
3377                node: "fix",
3378                prompts: &prompts,
3379                cache: cache.as_deref(),
3380            };
3381            let (seat, out) =
3382                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3383            let agent_id = seat.agent.clone();
3384
3385            let mut fix = FixRecord {
3386                agent: agent_id,
3387                addressed: Vec::new(),
3388                rejected: Vec::new(),
3389                notes: String::new(),
3390                committed: false,
3391                failed: None,
3392                duration_ms: 0,
3393                continuation: None,
3394            };
3395            let mut continuation = ContinuationRecord::not_needed();
3396            let mut final_seat = seat.clone();
3397            match out {
3398                AgentOutcome::Ok(o) => {
3399                    fix.duration_ms = o.duration_ms;
3400                    let parsed = verdict::extract_json::<FixReport>(&o.text);
3401                    // A parsed report standing next to a command this same
3402                    // reply's own CLI never confirmed the exit status of is
3403                    // not a resolved answer — the identical `CommandEvidence`
3404                    // `state.jobs` renders, read here instead of only on
3405                    // display, per the completion judgment and the shown
3406                    // record needing to agree.
3407                    let incomplete_reason = match &parsed {
3408                        Ok(_) if has_unconfirmed_command(&o.commands) => Some(
3409                            "the reply parsed, but it reported a command whose own CLI never \
3410                             confirmed an exit status"
3411                                .to_owned(),
3412                        ),
3413                        Ok(_) => None,
3414                        Err(e) => Some(e.to_string()),
3415                    };
3416                    match incomplete_reason {
3417                        None => {
3418                            let report = parsed.expect("checked Ok above");
3419                            fix.addressed = report.addressed;
3420                            fix.rejected = report.rejected;
3421                            fix.notes =
3422                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
3423                        }
3424                        Some(reason) => {
3425                            let (resumed_seat, resolved, failure, cont) = self
3426                                .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
3427                                .await;
3428                            fix.duration_ms += cont.cumulative_wait_ms;
3429                            continuation = cont;
3430                            final_seat = resumed_seat;
3431                            match resolved {
3432                                Some(report) => {
3433                                    fix.addressed = report.addressed;
3434                                    fix.rejected = report.rejected;
3435                                    fix.notes = blind::sanitize_prose(
3436                                        &report.notes,
3437                                        &self.state.config.blind,
3438                                    );
3439                                }
3440                                None => fix.failed = failure,
3441                            }
3442                        }
3443                    }
3444                }
3445                // The CLI's raw error JSON is not a fix report to parse.
3446                AgentOutcome::Dropped(o) => {
3447                    fix.duration_ms = o.duration_ms;
3448                    let why = o
3449                        .dropped
3450                        .as_ref()
3451                        .map(|d| d.why.as_str())
3452                        .unwrap_or("the CLI ended the stream without delivering its answer");
3453                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3454                }
3455                AgentOutcome::Quota(o) => {
3456                    self.state.quota.push(QuotaLoss {
3457                        seat: final_seat.key.clone(),
3458                        node: "fix".to_owned(),
3459                        at: Timestamp::now(),
3460                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3461                    });
3462                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3463                }
3464                AgentOutcome::Failed(e) => fix.failed = Some(e),
3465            }
3466            fix.continuation = Some(continuation);
3467            self.state.seats.insert(final_seat.key.clone(), final_seat);
3468            git::commit_all(
3469                &winner.worktree,
3470                &format!("magi: review round {round} fixes (uncommitted work)"),
3471            )
3472            .await
3473            .ok();
3474            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
3475            fix.committed = after != before;
3476            // Judged by what `git` says moved against base, never by the
3477            // fixer's own `addressed`/`rejected` count — see
3478            // `ReviewRound::progressed`. Propagated with `?`, the same as the
3479            // `patch` snapshot above: swallowing this error would default
3480            // `diff_after` to empty, which almost always differs from a
3481            // non-empty `patch` and reads as "progressed" — exactly backwards
3482            // for a `git` failure the stagnation check cannot see through.
3483            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
3484            let progressed = diff_after != patch;
3485            let commit_note = if fix.committed {
3486                "committed"
3487            } else {
3488                "NO new commit"
3489            };
3490            let tree_note = if progressed {
3491                "changed vs base"
3492            } else {
3493                "unchanged vs base"
3494            };
3495            self.state.event(
3496                "fix",
3497                match &fix.failed {
3498                    // Distinct on purpose from "0 addressed, 0 rejected": the
3499                    // fixer's own diff still landed (blocking counts do keep
3500                    // falling round over round), only its adoption report did
3501                    // not come back, so this must never read like every
3502                    // finding was reviewed and declined.
3503                    Some(reason) => {
3504                        format!(
3505                            "round {round}: fixer's adoption report was lost ({reason}); \
3506                             {commit_note}, tree {tree_note}"
3507                        )
3508                    }
3509                    None => format!(
3510                        "round {round}: {} addressed, {} rejected, {commit_note}, tree \
3511                         {tree_note}{}",
3512                        fix.addressed.len(),
3513                        fix.rejected.len(),
3514                        if continuation.outcome == ContinuationOutcome::Resumed {
3515                            format!(
3516                                " (adoption report recovered after {} continuation(s))",
3517                                continuation.attempts
3518                            )
3519                        } else {
3520                            String::new()
3521                        },
3522                    ),
3523                },
3524            );
3525            round_record.fix = Some(fix);
3526            round_record.progressed = progressed;
3527            self.state.reviews.push(round_record);
3528            self.state.save()?;
3529
3530            // The fixer's own report never came back this round, even after
3531            // `continue_fix_report`'s own budget was spent on it — not an
3532            // ordinary "no report" (dropped stream, quota, plain failure),
3533            // which already reads that way and is left to the existing round
3534            // budget. Stopping here, rather than opening another round, is
3535            // what keeps a next reviewer/fixer wave from ever being
3536            // dispatched onto `winner.worktree` while whatever the seat's
3537            // last call may still have running there is unaccounted for: no
3538            // process liveness check exists (and none is being added — see
3539            // AGENTS.md/this task's own scope), so the only way to honour
3540            // "nothing starts before a valid report returns" is to not start
3541            // anything further on this worktree from this run at all.
3542            if matches!(
3543                continuation.outcome,
3544                ContinuationOutcome::Exhausted
3545                    | ContinuationOutcome::QuotaLost
3546                    | ContinuationOutcome::NoSession
3547            ) {
3548                return self
3549                    .stop_reviewing(
3550                        "the fixer's adoption report never came back, even after resuming its \
3551                         own seat; refusing to start another round against the same worktree \
3552                         while that is unresolved",
3553                        &shell,
3554                        &winner.worktree,
3555                    )
3556                    .await;
3557            }
3558
3559            prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
3560
3561            let streak = self
3562                .state
3563                .reviews
3564                .iter()
3565                .rev()
3566                .take_while(|r| !r.progressed)
3567                .count();
3568            if streak >= STAGNANT_LIMIT {
3569                return self
3570                    .stop_reviewing(
3571                        &format!(
3572                            "the tree has not moved against base for {streak} round(s) in a row"
3573                        ),
3574                        &shell,
3575                        &winner.worktree,
3576                    )
3577                    .await;
3578            }
3579        }
3580        Ok(())
3581    }
3582
3583    /// Decide, from the last recorded round's own verification, whether
3584    /// stopping the review loop is a hand-off or a genuine block.
3585    ///
3586    /// Called once the loop has given up trying — the round budget is spent,
3587    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
3588    /// findings still open, never while a round is still clean or the
3589    /// incomplete-panel case handled inline above. Gate and e2e are facts
3590    /// about the tree; a lingering review finding is an opinion, and this
3591    /// workload's own `magi stats` puts reviewer precision low enough
3592    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
3593    /// must not by itself stand between a green, verified change and the
3594    /// human who decides what to do with it. A red e2e is not an opinion, so
3595    /// that case still blocks, with the failing command and a tail of its
3596    /// output recorded here rather than left in `run.json` for someone to go
3597    /// find.
3598    ///
3599    /// A round that deferred its own e2e (see [`Config::graph`]'s
3600    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
3601    /// only because nothing ran, and treating an empty list as a passing one
3602    /// here is exactly the "deferred painted green" bug this function exists
3603    /// to not have. When the last round deferred, this makes the real run —
3604    /// on the actual worktree this loop is about to stop touching — before
3605    /// deciding anything.
3606    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
3607        let round_idx = self.state.reviews.len() - 1;
3608        let needs_catchup_run = {
3609            let last = &self.state.reviews[round_idx];
3610            last.e2e.is_empty() && last.e2e_deferred
3611        };
3612        if needs_catchup_run {
3613            let round = self.state.reviews[round_idx].round;
3614            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3615            let commands = self.state.config.verify.e2e.clone();
3616            let verified_head = git::rev_parse(worktree, "HEAD").await?;
3617            let cache_dir = self.state.config.cache_dir();
3618            let context =
3619                format!("round {round}: deferred e2e, now catching up before the final decision");
3620            let (outcomes, verify_retried) = with_cache_lease(
3621                &mut self.state,
3622                cache_dir.as_deref(),
3623                "e2e",
3624                "e2e",
3625                worktree,
3626                &verified_head,
3627                timeout,
3628                &context,
3629                |state, budget| {
3630                    let shell = shell.to_vec();
3631                    let commands = commands.clone();
3632                    let context = context.clone();
3633                    async move {
3634                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
3635                            .await
3636                    }
3637                },
3638            )
3639            .await;
3640            // The catch-up run never actually happened - the shared build
3641            // cache could not be acquired or confirmed fresh in time (see
3642            // `CommandOutcome::resource_blocked`'s own doc) - so this round's
3643            // `e2e`/`e2e_deferred` are left exactly as they were:
3644            // `needs_catchup_run` above still reads true the next time this
3645            // is reached, and the round stays deferred rather than recording
3646            // contention as a red e2e and blocking the run on it.
3647            if verify_inconclusive(&outcomes) {
3648                self.state.save()?;
3649                return Ok(());
3650            }
3651            let last = &mut self.state.reviews[round_idx];
3652            last.e2e = outcomes;
3653            last.verify_retried = verify_retried;
3654            last.e2e_deferred = false;
3655            if verified_head != last.head {
3656                last.verified_head = Some(verified_head);
3657            }
3658        }
3659        let last = &self.state.reviews[round_idx];
3660        let red: Vec<String> = last
3661            .e2e
3662            .iter()
3663            .filter(|o| !o.ok())
3664            .map(|o| {
3665                format!(
3666                    "`{}` -> {:?}\n{}",
3667                    o.command,
3668                    o.code,
3669                    tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3670                )
3671            })
3672            .collect();
3673        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
3674
3675        if red.is_empty() {
3676            self.state.event(
3677                "review",
3678                format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
3679            );
3680            self.state.status = RunStatus::Gating;
3681        } else {
3682            self.state
3683                .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
3684            self.state.status = RunStatus::Blocked;
3685        }
3686        self.state.save()?;
3687        Ok(())
3688    }
3689
3690    // ----------------------------------------------------------------- gate
3691
3692    async fn gate(&mut self) -> Result<()> {
3693        // Judged by the review record itself, not by `status`: a solo
3694        // candidate's `judge`/`deliberate` skip rewrites `status` on every
3695        // reentry (see `judge`), and trusting it here is exactly how a run
3696        // that exhausted its review budget got gated and merged a second
3697        // time around. `review_conclusion` recomputes the review loop's own
3698        // verdict from the round records themselves — `Gating` for a clean
3699        // round or a hand-off (see `stop_reviewing`), anything else means the
3700        // loop is still going or genuinely blocked.
3701        // A base the winner could not be replayed onto is a decision, not a
3702        // round: there is no landing tree to gate. Read as its own record for
3703        // the same reason the review verdict is.
3704        if self.state.status == RunStatus::Failed
3705            || self
3706                .state
3707                .base_sync
3708                .as_ref()
3709                .is_some_and(|s| s.conflict.is_some())
3710            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3711                != Some(RunStatus::Gating)
3712        {
3713            return Ok(());
3714        }
3715        if self.state.gate_ran {
3716            // `review_loop` derives its conclusion from the clean review
3717            // record on every reentry and therefore puts a completed run back
3718            // in `Gating`. A recorded gate is a stronger, terminal fact:
3719            // retain its original command output (or lack of any, for a repo
3720            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
3721            // doc) and restore `Blocked` on a real failure rather than
3722            // pretending the command is still running or running it a second
3723            // time. `gate_ran == false` remains the only shape — unattempted,
3724            // or a resource-blocked retry — that may still need to execute a
3725            // command.
3726            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
3727                self.state.status = RunStatus::Blocked;
3728                self.state.save()?;
3729            }
3730            return Ok(());
3731        }
3732        let Some(winner) = self.state.winner().cloned() else {
3733            return Ok(());
3734        };
3735        self.state.status = RunStatus::Gating;
3736        let shell = self.state.config.shell();
3737        let gate_commands = self.state.config.verify.gate.clone();
3738        // Zero commands has nothing to run and nothing that could touch the
3739        // shared build cache, so it never needs a lease: `Config::cache_dir`
3740        // is derived from `verify.e2e` too, so a repo with no `verify.gate`
3741        // commands but a `CARGO_TARGET_DIR`-using `verify.e2e` would
3742        // otherwise queue behind an unrelated run's lease and come back
3743        // resource-blocked - `gate_ran` would stay false on nothing but
3744        // cache contention, for a step that had nothing to check in the
3745        // first place.
3746        let outcomes = if gate_commands.is_empty() {
3747            Vec::new()
3748        } else {
3749            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3750            let cache_dir = self.state.config.cache_dir();
3751            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
3752            let (outcomes, _) = with_cache_lease(
3753                &mut self.state,
3754                cache_dir.as_deref(),
3755                "gate",
3756                "gate",
3757                &winner.worktree,
3758                &head,
3759                timeout,
3760                "final gate",
3761                |_state, budget| {
3762                    let shell = shell.clone();
3763                    let gate_commands = gate_commands.clone();
3764                    let worktree = winner.worktree.clone();
3765                    async move {
3766                        let (outcomes, timed_out_pids) =
3767                            run_commands(&shell, &gate_commands, &worktree, budget).await;
3768                        (outcomes, false, timed_out_pids)
3769                    }
3770                },
3771            )
3772            .await;
3773            outcomes
3774        };
3775        if outcomes.is_empty() {
3776            // Nothing configured to check — distinct from every other
3777            // silence in this run's event log, since an empty `gate` alone
3778            // no longer says whether the gate ran at all (see
3779            // `RunState::gate_ran`'s own doc).
3780            self.state.event(
3781                "gate",
3782                "no gate commands configured; nothing to check, passing",
3783            );
3784        }
3785        for o in &outcomes {
3786            self.state.event(
3787                "gate",
3788                format!(
3789                    "`{}` -> {}",
3790                    o.command,
3791                    if o.ok() {
3792                        "pass".to_owned()
3793                    } else {
3794                        format!(
3795                            "FAIL ({:?})\n{}",
3796                            o.code,
3797                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3798                        )
3799                    }
3800                ),
3801            );
3802        }
3803        // A resource-blocked outcome means the gate command never actually
3804        // ran - the shared build cache could not be acquired or confirmed
3805        // fresh in time - which is evidence about the machine, not about the
3806        // tree (see `CommandOutcome::resource_blocked`'s own doc). Recording
3807        // it as a red gate would mark a run `Blocked` on nothing but
3808        // contention magi has already logged above; leaving `self.state.gate`
3809        // empty and `self.state.gate_ran` false instead keeps the shape this
3810        // function already treats as "still needs to run" (see the
3811        // early-return above), so the next call retries the command rather
3812        // than concluding anything.
3813        if verify_inconclusive(&outcomes) {
3814            self.state.save()?;
3815            return Ok(());
3816        }
3817        let passed = outcomes.iter().all(CommandOutcome::ok);
3818        self.state.gate = outcomes;
3819        self.state.gate_ran = true;
3820        if !passed {
3821            self.state.status = RunStatus::Blocked;
3822            self.state.event("gate", "gate failed; not merging");
3823        }
3824        self.state.save()?;
3825        Ok(())
3826    }
3827
3828    // ---------------------------------------------------------------- merge
3829
3830    async fn merge(&mut self) -> Result<()> {
3831        // Same reasoning as `gate`: ask the review and gate records directly
3832        // rather than `status`, which a solo-candidate `judge`/`deliberate`
3833        // skip can rewrite on reentry to something that no longer says
3834        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
3835        // so a hand-off (open findings, green verification) reaches merge
3836        // exactly like a genuinely clean round does.
3837        //
3838        // A run resumed mid-`land` never reaches here at all: `execute`
3839        // recognises `RunStatus::Landing` before it even calls `prep`, and
3840        // routes straight to `run_land` instead. That has to happen a level
3841        // up from this function, not with a check in here, because
3842        // `review_loop`'s own status recomputation (see its doc) runs
3843        // *before* `merge` on every reentry and would otherwise overwrite
3844        // the `Landing` marker with `Gating` before this node ever saw it.
3845        if self
3846            .state
3847            .base_sync
3848            .as_ref()
3849            .is_some_and(|s| s.conflict.is_some())
3850            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3851                != Some(RunStatus::Gating)
3852            // `gate_ran == false` is not "passed" - `gate` leaves it false
3853            // both before it has ever run and when its last attempt was
3854            // resource-blocked (see `Runner::gate`'s own doc), and neither is
3855            // permission to merge on nothing but the review record. Only a
3856            // gate that actually ran - zero commands configured and
3857            // vacuously passed, or one or more that all exited 0 - may
3858            // proceed; `RunState::gate_status` is the single place that
3859            // reading is computed.
3860            || !self.state.gate_status().ok()
3861        {
3862            return Ok(());
3863        }
3864        // This node's own record, not `status`: `status == Ready` is not
3865        // unique to the harmless `MergeMode::None` path this line was
3866        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
3867        // run's PR was closed without merging — and on that run `mode` is
3868        // still `Pr`, so a reentry that fell through here would push and
3869        // open a second pull request. `self.state.merge` is set exactly once
3870        // this node (or `land`) has already produced a verdict, under every
3871        // mode, which is what "already done" actually means here.
3872        if self.state.merge.is_some() {
3873            return Ok(());
3874        }
3875        let Some(winner) = self.state.winner().cloned() else {
3876            return Ok(());
3877        };
3878        let repo = self.state.repo.clone();
3879        let base = self.state.base_branch.clone();
3880        let mode = self.state.config.merge.mode;
3881        let style = self.state.config.merge.style;
3882        let message = pr_body(&self.state, winner.label);
3883
3884        let outcome = match mode {
3885            MergeMode::None => MergeOutcome {
3886                mode,
3887                ok: true,
3888                detail: manual_merge_command(style, &repo, &winner.branch, &message),
3889            },
3890            MergeMode::Local => {
3891                let on = git::current_branch(&repo).await?;
3892                if on.as_deref() != Some(base.as_str()) {
3893                    MergeOutcome {
3894                        mode,
3895                        ok: false,
3896                        detail: format!(
3897                            "{} has {} checked out, not the base branch {base}",
3898                            repo.display(),
3899                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
3900                        ),
3901                    }
3902                } else if !git::is_clean(&repo).await? {
3903                    MergeOutcome {
3904                        mode,
3905                        ok: false,
3906                        detail: format!("{} is dirty; refusing to merge", repo.display()),
3907                    }
3908                } else {
3909                    let out = match style {
3910                        MergeStyle::Merge => {
3911                            git::merge_no_ff(&repo, &winner.branch, &message).await?
3912                        }
3913                        MergeStyle::Squash => {
3914                            git::merge_squash(&repo, &winner.branch, &message).await?
3915                        }
3916                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
3917                    };
3918                    MergeOutcome {
3919                        mode,
3920                        ok: out.ok(),
3921                        detail: if out.ok() { out.stdout } else { out.stderr },
3922                    }
3923                }
3924            }
3925            MergeMode::Pr => {
3926                let remote = self.state.config.merge.remote.clone();
3927                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
3928                if !pushed.ok() {
3929                    MergeOutcome {
3930                        mode,
3931                        ok: false,
3932                        detail: pushed.stderr,
3933                    }
3934                } else {
3935                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
3936                    match out {
3937                        Ok(url) => MergeOutcome {
3938                            mode,
3939                            ok: true,
3940                            detail: url,
3941                        },
3942                        Err(e) => MergeOutcome {
3943                            mode,
3944                            ok: false,
3945                            detail: e.to_string(),
3946                        },
3947                    }
3948                }
3949            }
3950        };
3951
3952        self.state.status = match (mode, outcome.ok) {
3953            (MergeMode::None, _) => RunStatus::Ready,
3954            (_, true) => RunStatus::Merged,
3955            (_, false) => RunStatus::Blocked,
3956        };
3957        self.state.event(
3958            "merge",
3959            format!(
3960                "{:?}: {}",
3961                mode,
3962                outcome.detail.lines().next().unwrap_or("")
3963            ),
3964        );
3965        self.state.merge = Some(outcome);
3966        self.state.save()?;
3967
3968        // The PR is open and the run would historically stop here, leaving the
3969        // operator to watch checks, feed review comments back to a fixer, and
3970        // merge. That was done by hand six times in one session before this
3971        // existed. Opt-in, because merging is the one irreversible thing magi
3972        // can do to a repository.
3973        if self.state.config.graph.land
3974            && mode == MergeMode::Pr
3975            && self.state.status == RunStatus::Merged
3976        {
3977            self.run_land().await?;
3978        }
3979        // `run_land` may have left `status` at `Landing` - still waiting on
3980        // CI or the owner's approval, not actually settled - so this has to
3981        // read whatever `status` ended up as here, not the `Merged` this
3982        // function set a few lines up.
3983        self.settle_questions();
3984        Ok(())
3985    }
3986
3987    /// Enter `land`.
3988    ///
3989    /// Shared between a fresh run's first pass through [`Runner::merge`] and
3990    /// a resumed run's re-entry. `land::land` itself is what serialises the
3991    /// two git-mutating moments inside the loop — the rebase push and
3992    /// `gh pr merge` — per repository (see its own doc); nothing here needs
3993    /// to hold a lock across the whole call, and doing so would serialise
3994    /// this run's CI wait against a *different* run's land-approval resume
3995    /// in the same repository, which is exactly the "must not wait on
3996    /// another task" property the daemon's slot-freeing exists to give.
3997    async fn run_land(&mut self) -> Result<()> {
3998        let url = self
3999            .state
4000            .merge
4001            .as_ref()
4002            .map(|m| m.detail.clone())
4003            .unwrap_or_default();
4004        let url = url.lines().next().unwrap_or("").trim().to_owned();
4005        if !url.starts_with("http") {
4006            return Ok(());
4007        }
4008        // A land failure is not a lost run: the work is on a branch and the
4009        // pull request is open, which is exactly where a human takes over.
4010        match land::land(&mut self.state, &url).await {
4011            Ok(pr) if self.state.parked => {
4012                // `land` already saved the parked marker; nothing here
4013                // overrides `status` back to a terminal value while an
4014                // approval is still outstanding.
4015                let _ = pr;
4016            }
4017            Ok(pr) => {
4018                self.state.status = match pr.state {
4019                    land::PrLifecycle::Merged => RunStatus::Merged,
4020                    _ => RunStatus::Blocked,
4021                };
4022                // Downstream of a confirmed merge only - see
4023                // `bump::should_release_bump`'s own doc for why this one
4024                // check covers all three of `land`'s success paths.
4025                // Best-effort: the run already landed, so a failure here
4026                // (the decision call, `gh`, `cargo`) is recorded and never
4027                // turns a landed run into a failed one.
4028                if bump::should_release_bump(self.state.status)
4029                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
4030                {
4031                    self.state
4032                        .event("bump", format!("release bump skipped: {e:#}"));
4033                }
4034                self.state.save()?;
4035            }
4036            Err(e) => {
4037                self.state.status = RunStatus::Blocked;
4038                self.state.event("land", format!("gave up: {e}"));
4039                self.state.save()?;
4040            }
4041        }
4042        Ok(())
4043    }
4044
4045    // -------------------------------------------------------------- helpers
4046
4047    /// Fetch or create a seat, keeping its conversation across nodes.
4048    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
4049        if let Some(existing) = self.state.seats.get(key)
4050            && existing.agent == agent
4051        {
4052            return existing.clone();
4053        }
4054        let fresh = SeatState::new(key, agent, self.state.seed);
4055        self.state.seats.insert(key.to_owned(), fresh.clone());
4056        fresh
4057    }
4058
4059    /// A candidate rendered for judging, with the leak policy applied.
4060    fn view(&self, c: &Candidate) -> CandidateView {
4061        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
4062            .unwrap_or_default();
4063        let (patch, _) = blind::sanitize_patch(
4064            &format!("candidate {} patch", c.label),
4065            &raw,
4066            &self.state.config.blind,
4067        );
4068        CandidateView {
4069            label: c.label,
4070            branch: c.branch.clone(),
4071            summary: c.summary.clone(),
4072            stat: c.stat.clone(),
4073            patch,
4074        }
4075    }
4076
4077    /// The full candidate set as prompt text, for seats with no live session.
4078    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
4079        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
4080        prompt::judge(
4081            "(see above)",
4082            &views,
4083            self.roles.judges.len(),
4084            base_short,
4085            "en",
4086        )
4087    }
4088
4089    /// Anonymised transcript for judge `self_idx`.
4090    ///
4091    /// The initial rankings are always the opening statements. Seeding them
4092    /// only when no turn had been taken yet meant every judge after the first
4093    /// argued against a single voice instead of against the actual split — the
4094    /// disagreement is the information, so it is always on the table.
4095    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
4096        let mut turns = Vec::new();
4097        for j in &self.state.judgements {
4098            if j.ranking.is_empty() {
4099                continue;
4100            }
4101            let reasons = j
4102                .reasons
4103                .iter()
4104                .map(|(k, v)| format!("- {k}: {v}"))
4105                .collect::<Vec<_>>()
4106                .join("\n");
4107            turns.push(Turn {
4108                who: format!("Judge {} (opening ranking)", j.judge),
4109                is_self: j.judge == self_idx + 1,
4110                body: format!(
4111                    "Ranked {}{}{reasons}",
4112                    j.ranking.iter().collect::<String>(),
4113                    if reasons.is_empty() {
4114                        ""
4115                    } else {
4116                        ", because:\n"
4117                    }
4118                ),
4119            });
4120        }
4121        for t in self
4122            .state
4123            .deliberation
4124            .iter()
4125            .flat_map(|r| r.turns.iter())
4126            .chain(current)
4127        {
4128            turns.push(Turn {
4129                who: format!("Judge {}", t.judge),
4130                is_self: t.judge == self_idx + 1,
4131                body: t.body.clone(),
4132            });
4133        }
4134        turns
4135    }
4136}
4137
4138/// Does this seat still hold the context a follow-up prompt would rely on?
4139fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
4140    agent::has_session(spec.kind, seat, sessions)
4141}
4142
4143/// Did this reply report running a command whose own CLI never confirmed an
4144/// exit status?
4145///
4146/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
4147/// command *finished* (see that type's own doc), so this can only be `true`
4148/// for a command whose completion event carried no readable exit code — not
4149/// for one that simply is not mentioned at all. That is the one signal this
4150/// crate can read, from the same record `state.jobs` renders, about a reply
4151/// standing next to work its own CLI cannot vouch for finishing; it is
4152/// deliberately not a check on the exit code's *value* (a fixer legitimately
4153/// runs a command that fails mid-iteration before it succeeds) and not a
4154/// guess at a command still running in the background (which emits no event
4155/// at all, and so leaves no evidence here to find).
4156fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
4157    commands.iter().any(|c| c.exit_code.is_none())
4158}
4159
4160fn short(commit: &str) -> String {
4161    commit.chars().take(7).collect()
4162}
4163
4164fn make_executable(path: &Path) -> Result<()> {
4165    #[cfg(unix)]
4166    {
4167        use std::os::unix::fs::PermissionsExt as _;
4168        let mut perms = std::fs::metadata(path)?.permissions();
4169        perms.set_mode(0o755);
4170        std::fs::set_permissions(path, perms)?;
4171    }
4172    #[cfg(not(unix))]
4173    {
4174        let _ = path;
4175    }
4176    Ok(())
4177}
4178
4179/// What every seat in one batch shares: where the answers are attributed, the
4180/// prompt overlay they inherit, and the build cache they are told to use.
4181///
4182/// A struct rather than four more parameters: `wave` also needs the run's
4183/// state (to record who is answering right now) and the attempt number, and
4184/// eight positional arguments is both unreadable and a clippy error.
4185struct WaveCtx<'a> {
4186    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
4187    /// paid for it.
4188    run: &'a str,
4189    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
4190    node: &'a str,
4191    prompts: &'a Prompts,
4192    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
4193    cache: Option<&'a Path>,
4194}
4195
4196/// Run one job, honouring the parallelism budget.
4197async fn run_one(
4198    job: SeatJob,
4199    sem: Arc<Semaphore>,
4200    ctx: &WaveCtx<'_>,
4201    state: &mut RunState,
4202    attempt: usize,
4203) -> (SeatState, AgentOutcome) {
4204    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
4205        .await
4206        .pop()
4207        .expect("one job in, one result out");
4208    (seat, out)
4209}
4210
4211/// Run every job concurrently, capped by the semaphore, preserving order.
4212///
4213/// Every seat in the batch is recorded into [`RunState::active`] before the
4214/// wave starts and cleared as each answer lands, so the run's own record says
4215/// who is still being waited on rather than only who finished.
4216async fn wave(
4217    jobs: Vec<SeatJob>,
4218    sem: Arc<Semaphore>,
4219    ctx: &WaveCtx<'_>,
4220    state: &mut RunState,
4221    attempt: usize,
4222) -> Vec<(usize, SeatState, AgentOutcome)> {
4223    let WaveCtx {
4224        run,
4225        node,
4226        prompts,
4227        cache,
4228    } = *ctx;
4229    for job in &jobs {
4230        state.seat_started(node, &job.seat.key, job.timeout, attempt);
4231    }
4232    if let Err(e) = state.save() {
4233        // A failed persist of "who is answering right now" must not abort the
4234        // wave: the seats are already being asked, and the alternative is
4235        // losing the answers to save a status line nobody may even be
4236        // watching.
4237        tracing::warn!("could not persist in-progress seats: {e:#}");
4238    }
4239    // Hold the shared build cache's lease for the whole batch, not per job:
4240    // several candidates (an implement wave) or a fixer legitimately share
4241    // one cache concurrently within this run, and that stays untouched — a
4242    // single lease taken once for the whole wave and released once it is
4243    // done is what stops a *different* borrower (another run's own wave, its
4244    // e2e/gate, a human's `magi review`) from interleaving a build into the
4245    // same directory while this one is in flight. Best-effort, not
4246    // all-or-nothing: a wave that cannot get the lease within its own
4247    // longest job's budget still runs — an hour of paid implementer calls is
4248    // not thrown away over cache contention — but every write-allowed seat
4249    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
4250    // below), the same fallback a read-only seat always gets, rather than
4251    // building into a directory this run was never granted. The identity
4252    // record is still invalidated below either way, so the next tracked
4253    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
4254    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
4255    let wait_started = Instant::now();
4256    let cache_guard = if let Some(cache_dir) = cache {
4257        if jobs_had_a_writer {
4258            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
4259            let budget = jobs
4260                .iter()
4261                .map(|j| j.timeout)
4262                .max()
4263                .unwrap_or(Duration::from_secs(60));
4264            acquire_cache_lease(state, cache_dir, &owner, budget, node)
4265                .await
4266                .ok()
4267        } else {
4268            None
4269        }
4270    } else {
4271        None
4272    };
4273    // Carved out of each job's own budget, not added on top of it: a seat
4274    // that waited behind the lease must not also get its full timeout
4275    // afterward, or a run contended on the cache could double the time it
4276    // spends per wave. `saturating_sub` floors at zero rather than
4277    // wrapping - a job whose whole budget was spent waiting starts with
4278    // none left, which is the honest number, not a free minimum.
4279    let waited_for_lease = wait_started.elapsed();
4280    let mut set = tokio::task::JoinSet::new();
4281    let overlay = prompts.overlay(node);
4282    for (i, mut job) in jobs.into_iter().enumerate() {
4283        job.timeout = job.timeout.saturating_sub(waited_for_lease);
4284        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
4285        if cache.is_some() {
4286            job.prompt.push('\n');
4287            job.prompt
4288                .push_str(&prompt::build_cache_note(node, job.allow_write));
4289        }
4290        let sem = Arc::clone(&sem);
4291        let run = run.to_owned();
4292        let node = node.to_owned();
4293        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
4294        // `prompt::build_cache_note`'s doc for why setting it anyway is
4295        // exactly how a sandboxed reviewer's write refusal got reported as a
4296        // defect in the patch, not a property of its own seat. And a
4297        // write-allowed one is handed it only when the lease above was
4298        // actually acquired: a wave that could not get it (`cache_guard` is
4299        // `None`, see its own comment) must not send seats to build into a
4300        // directory this run does not hold - that is the exact concurrent,
4301        // unmanaged-write race this module exists to prevent, not something
4302        // "proceeding anyway" is allowed to reintroduce.
4303        let cache = cache
4304            .filter(|_| job.allow_write && cache_guard.is_some())
4305            .map(Path::to_path_buf);
4306        set.spawn(async move {
4307            let _permit = sem.acquire().await;
4308            let mut seat = job.seat;
4309            let out = agent::invoke(
4310                &job.spec,
4311                &mut seat,
4312                &Invocation {
4313                    cwd: &job.cwd,
4314                    prompt: &job.prompt,
4315                    timeout: job.timeout,
4316                    allow_write: job.allow_write,
4317                    sessions: job.sessions,
4318                    artifacts: &job.artifacts,
4319                    stem: &job.stem,
4320                    run: &run,
4321                    node: &node,
4322                    cache_dir: cache.as_deref(),
4323                    attachments: &[],
4324                },
4325            )
4326            .await;
4327            let out = match out {
4328                Ok(o) if o.usable() => AgentOutcome::Ok(o),
4329                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
4330                // Billed work the CLI failed to hand over is not an ordinary
4331                // failure, but its text is the CLI's raw error JSON, not an
4332                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
4333                // read it as one by forgetting to check. `usable()` is always
4334                // false here (dropped implies an empty response), so this has
4335                // to be checked before the catch-all `Failed` below or the
4336                // one shape this exists for is lost with the rest.
4337                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
4338                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
4339                Ok(o) => AgentOutcome::Failed(format!(
4340                    "exited with {:?} and no usable output",
4341                    o.exit_code
4342                )),
4343                Err(e) => AgentOutcome::Failed(e.to_string()),
4344            };
4345            (i, seat, out)
4346        });
4347    }
4348    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
4349    while let Some(joined) = set.join_next().await {
4350        let (i, seat, out) = match joined {
4351            Ok(v) => v,
4352            // No seat to clear: a panicked task never reported which one it
4353            // was. The defensive sweep below this loop is what stops that
4354            // seat's `active` entry from surviving forever.
4355            Err(e) => {
4356                tracing::error!("agent task panicked: {e}");
4357                continue;
4358            }
4359        };
4360        state.seat_finished(&seat.key);
4361        record_jobs(state, node, &seat.key, &out);
4362        if let Err(e) = state.save() {
4363            tracing::warn!("could not persist a seat's completion: {e:#}");
4364        }
4365        if collected.len() <= i {
4366            collected.resize_with(i + 1, || None);
4367        }
4368        collected[i] = Some((i, seat, out));
4369    }
4370    // Belt-and-braces for the panic branch above: every seat this exact batch
4371    // started shares this `(node, attempt)` pair, and every seat that finished
4372    // normally already cleared itself, so anything left tagged with it here
4373    // can only be a panicked task's leftover. Cleared unconditionally rather
4374    // than left to read as still answering forever.
4375    if state
4376        .active
4377        .values()
4378        .any(|a| a.node == node && a.attempt == attempt)
4379    {
4380        state
4381            .active
4382            .retain(|_, a| !(a.node == node && a.attempt == attempt));
4383        if let Err(e) = state.save() {
4384            tracing::warn!("could not persist the end of a wave: {e:#}");
4385        }
4386    }
4387    // Whether or not the lease above was actually held, several worktrees
4388    // may just have built into the cache with nothing here able to name one
4389    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
4390    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
4391    // clean it might not have strictly needed; trusting a stale match would
4392    // cost it a wrong answer.
4393    if let Some(cache_dir) = cache
4394        && jobs_had_a_writer
4395    {
4396        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
4397    }
4398    if let Some(guard) = cache_guard {
4399        guard.release();
4400    }
4401    collected.into_iter().flatten().collect()
4402}
4403
4404/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
4405/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
4406/// this is data collection, not the fix-specific completion contract in
4407/// [`Runner::continue_fix_report`], and applies regardless of which node
4408/// asked.
4409///
4410/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
4411/// evidence from; `Failed` does not, and correctly contributes nothing — a
4412/// timeout or crash is not itself evidence about a command the seat may have
4413/// started.
4414fn record_jobs(state: &mut RunState, node: &str, seat: &str, out: &AgentOutcome) {
4415    let commands: &[agent::CommandEvidence] = match out {
4416        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
4417        AgentOutcome::Failed(_) => &[],
4418    };
4419    let checked_at = Timestamp::now();
4420    for c in commands {
4421        state.jobs.push(JobRecord {
4422            node: node.to_owned(),
4423            seat: seat.to_owned(),
4424            id: c.id.clone(),
4425            description: c.description.clone(),
4426            checked_at,
4427            status: match c.exit_code {
4428                Some(0) => JobStatus::Completed,
4429                Some(_) => JobStatus::Failed,
4430                None => JobStatus::Unknown,
4431            },
4432            exit_code: c.exit_code,
4433            result_summary: c.result_summary.clone(),
4434            source: c.source.clone(),
4435        });
4436    }
4437}
4438
4439/// Is a review round clean, given how many reviewer seats answered against
4440/// how many the round expected?
4441///
4442/// A seat that never answered (timeout, crash, unparsable output) is not a
4443/// seat that read the patch and found nothing — treating it as such is
4444/// exactly the bug this function exists to close. Under the default `block`
4445/// policy a missing seat can never be clean; `warn` still requires the seats
4446/// that *did* answer to have found nothing blocking and verification to be
4447/// green.
4448///
4449/// `quota_missing` narrows that `block` default for exactly one cause of
4450/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
4451/// a session limit lifts by the very next round buys nothing — the seat is
4452/// asked again with the same quota — so once every missing seat is accounted
4453/// for by a quota loss (and at least one seat *did* answer, so a decision has
4454/// something to rest on) the round is decided on the panel that could answer,
4455/// same as `warn` would. A panel that lost every seat to quota is not
4456/// decided here: `answered == 0` falls through to the existing `block`
4457/// fallback so a fully collapsed panel still waits rather than landing on no
4458/// review at all.
4459fn round_is_clean(
4460    blocking: usize,
4461    e2e_ok: bool,
4462    answered: usize,
4463    expected: usize,
4464    quota_missing: usize,
4465    policy: IncompleteReviewPolicy,
4466) -> bool {
4467    if blocking != 0 || !e2e_ok {
4468        return false;
4469    }
4470    if answered == expected || policy == IncompleteReviewPolicy::Warn {
4471        return true;
4472    }
4473    answered > 0 && expected - answered <= quota_missing
4474}
4475
4476/// The review loop's own conclusion, derived entirely from its persisted
4477/// round records and the round budget that produced them — never from
4478/// `status`, so a reentry (or `gate`/`merge` reading it independently)
4479/// recomputes the identical answer regardless of what an earlier node in the
4480/// same walk, or a previous walk, did to `status`.
4481///
4482/// `None` while more rounds remain to try, including when review never ran
4483/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
4484/// gone clean, or the budget is spent, or the tree has stopped moving (see
4485/// [`STAGNANT_LIMIT`]), the answer is one of two things:
4486///
4487/// - An incomplete panel that raised nothing is missing input, not a
4488///   verified tree — never a hand-off candidate, whatever verification said
4489///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
4490/// - Otherwise, green e2e on the last round hands off (see
4491///   [`Runner::stop_reviewing`]); red e2e blocks.
4492fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
4493    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
4494        return Some(RunStatus::Gating);
4495    }
4496    let last = reviews.last()?;
4497    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
4498    if reviews.len() < max_rounds && !stagnant {
4499        return None;
4500    }
4501    Some(if last.incomplete() && last.blocking == 0 {
4502        RunStatus::Blocked
4503    } else if last.e2e.iter().all(CommandOutcome::ok) {
4504        RunStatus::Gating
4505    } else {
4506        RunStatus::Blocked
4507    })
4508}
4509
4510/// How long a re-ask may take, given the budget the first attempt had.
4511///
4512/// A `nudged` retry is a request to restate an answer the seat has already
4513/// worked out: it carries no new work, so it does not deserve the original
4514/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
4515/// 133 seconds while a third sat for over ten minutes on a resumed session
4516/// holding 410 KB of prior output - and because the retry had inherited the
4517/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
4518/// judging round whose other seats were long finished.
4519///
4520/// A quarter of the budget, with a floor so that a deliberately short timeout
4521/// does not collapse to nothing. A retry that re-sends the whole prompt
4522/// (because the seat kept no context) is the original job again, and keeps the
4523/// original budget.
4524fn retry_budget(full: Duration, nudged: bool) -> Duration {
4525    if nudged {
4526        (full / 4).max(Duration::from_secs(120)).min(full)
4527    } else {
4528        full
4529    }
4530}
4531
4532/// Run a wave and parse each reply, re-asking the seats whose reply was
4533/// unusable.
4534///
4535/// The re-ask is a nudge rather than the whole prompt again when the seat still
4536/// holds its conversation, which is the difference between a cheap retry and
4537/// paying for the entire candidate set twice.
4538///
4539/// A seat that hits a rate limit is **not** re-asked: the same call will fail
4540/// the same way until the limit resets, so spending a retry attempt on it is
4541/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
4542/// like any other absent seat — the caller decides whether the panel still has
4543/// a quorum.
4544#[allow(clippy::too_many_arguments)]
4545async fn ask_json_wave<T>(
4546    jobs: Vec<SeatJob>,
4547    sem: Arc<Semaphore>,
4548    retries: usize,
4549    ctx: &WaveCtx<'_>,
4550    losses: &mut Vec<QuotaLoss>,
4551    state: &mut RunState,
4552    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
4553) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
4554where
4555    T: serde::de::DeserializeOwned + Send + 'static,
4556{
4557    let n = jobs.len();
4558    let originals: Vec<SeatJob> = jobs;
4559    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
4560    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
4561    let mut pending: Vec<usize> = (0..n).collect();
4562
4563    for attempt in 0..=retries {
4564        if pending.is_empty() {
4565            break;
4566        }
4567        let mut batch = Vec::with_capacity(pending.len());
4568        for &i in &pending {
4569            let src = &originals[i];
4570            // The prompt and the budget are one decision: a nudge restates
4571            // finished work, a re-sent prompt redoes it.
4572            let (prompt, timeout) = if attempt == 0 {
4573                (src.prompt.clone(), src.timeout)
4574            } else {
4575                let why = done[i]
4576                    .as_ref()
4577                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
4578                    .unwrap_or_else(|| "no parsable answer".to_owned());
4579                let nudge = prompt::nudge(&why);
4580                let nudged = has_context(&src.spec, &seats[i], src.sessions);
4581                let prompt = if nudged {
4582                    nudge
4583                } else {
4584                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
4585                };
4586                (prompt, retry_budget(src.timeout, nudged))
4587            };
4588            batch.push(SeatJob {
4589                spec: src.spec.clone(),
4590                seat: seats[i].clone(),
4591                cwd: src.cwd.clone(),
4592                prompt,
4593                timeout,
4594                allow_write: src.allow_write,
4595                sessions: src.sessions,
4596                artifacts: src.artifacts.clone(),
4597                stem: if attempt == 0 {
4598                    src.stem.clone()
4599                } else {
4600                    format!("{}-retry{attempt}", src.stem)
4601                },
4602            });
4603        }
4604
4605        if attempt > 0 {
4606            let seats_out: Vec<&str> = pending
4607                .iter()
4608                .map(|&i| originals[i].seat.key.as_str())
4609                .collect();
4610            state.event(
4611                ctx.node,
4612                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
4613            );
4614        }
4615        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
4616        let mut still = Vec::new();
4617        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
4618            seats[i] = seat;
4619            let (parsed, quota) = match out {
4620                AgentOutcome::Ok(o) => (
4621                    match verdict::extract_json::<T>(&o.text) {
4622                        Ok(v) => match validate(&v) {
4623                            Ok(()) => Ok((v, o)),
4624                            Err(e) => Err(e),
4625                        },
4626                        Err(e) => Err(e),
4627                    },
4628                    false,
4629                ),
4630                AgentOutcome::Quota(o) => {
4631                    losses.push(QuotaLoss {
4632                        seat: originals[i].seat.key.clone(),
4633                        node: ctx.node.to_owned(),
4634                        at: Timestamp::now(),
4635                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
4636                    });
4637                    (
4638                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
4639                        true,
4640                    )
4641                }
4642                // Not a parseable answer, but also not worth a special-cased
4643                // retry here: the nudge loop above already re-asks anything
4644                // that fails to parse, which is exactly what a dropped stream
4645                // needs. Just don't hand its raw error JSON to `extract_json`.
4646                AgentOutcome::Dropped(o) => {
4647                    let why = o
4648                        .dropped
4649                        .as_ref()
4650                        .map(|d| d.why.as_str())
4651                        .unwrap_or("the CLI ended the stream without delivering its answer");
4652                    (
4653                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
4654                        false,
4655                    )
4656                }
4657                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
4658            };
4659            let failed = parsed.is_err();
4660            done[i] = Some(parsed);
4661            // Do not re-ask a rate-limited seat (quota) — a retry is known to
4662            // fail the same way; and never re-ask a seat that already parsed.
4663            if failed && !quota {
4664                still.push(i);
4665            }
4666        }
4667        pending = still;
4668    }
4669
4670    seats
4671        .into_iter()
4672        .zip(done)
4673        .map(|(seat, res)| {
4674            (
4675                seat,
4676                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
4677            )
4678        })
4679        .collect()
4680}
4681
4682/// Acquire the shared build cache's lease, waiting out contention within
4683/// `budget` (never past it — see AGENTS.md's build-cache section on why an
4684/// unbounded wait is never acceptable).
4685///
4686/// A first, non-blocking check happens before ever waiting; if it finds the
4687/// lease busy, that fact is logged as a `verify` event *and* flushed with
4688/// [`RunState::save`] immediately — not only once the wait finally succeeds
4689/// or gives up — so a `magi show` run by a different process while this one
4690/// is still waiting reads a `run.json` that says so, rather than whatever it
4691/// looked like before the wait started. The same applies to the terminal
4692/// failure: logged and saved before this returns `Err`, so a caller that
4693/// could not get the lease at all still leaves a legible record of why.
4694async fn acquire_cache_lease(
4695    state: &mut RunState,
4696    cache_dir: &Path,
4697    owner: &crate::cache::Owner,
4698    budget: Duration,
4699    context: &str,
4700) -> Result<crate::cache::Guard> {
4701    let home = crate::run::home();
4702    let started = Instant::now();
4703    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
4704        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(g),
4705        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
4706        Err(e) => {
4707            state.event(
4708                "verify",
4709                format!("{context}: could not check the shared build cache: {e:#}"),
4710            );
4711            if let Err(e2) = state.save() {
4712                tracing::warn!("could not persist a cache-check failure: {e2:#}");
4713            }
4714            return Err(e);
4715        }
4716    };
4717    state.event(
4718        "verify",
4719        format!(
4720            "{context}: waiting for the shared build cache at {} ({})",
4721            cache_dir.display(),
4722            busy.describe()
4723        ),
4724    );
4725    if let Err(e) = state.save() {
4726        tracing::warn!("could not persist a cache wait: {e:#}");
4727    }
4728    let remaining = budget.saturating_sub(started.elapsed());
4729    match crate::cache::wait_for(&home, cache_dir, owner, remaining, Duration::from_secs(5)).await {
4730        Ok(g) => Ok(g),
4731        Err(e) => {
4732            state.event("verify", format!("{context}: {e:#}"));
4733            if let Err(e2) = state.save() {
4734                tracing::warn!("could not persist a cache wait timeout: {e2:#}");
4735            }
4736            Err(e)
4737        }
4738    }
4739}
4740
4741/// Run `body` — a verify command batch — while holding the shared build
4742/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
4743/// never interleave with another borrower's build against the same
4744/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
4745/// timeout, or a human's own `magi review`. See the `cache` module doc for
4746/// why this matters more than Cargo's own per-target locking covers — two
4747/// *different* worktrees building the same package name/version into one
4748/// cache directory is a staleness bug, not a lock contention one.
4749///
4750/// The wait for the lease is carved out of `budget`, never on top of it —
4751/// `body` is handed whatever is left, so a caller's own node timeout is the
4752/// only clock involved, exactly what AGENTS.md's build-cache section asks
4753/// for ("never an unbounded wait"). When `cache_dir` is `None` — no shared
4754/// cache configured at all — this is a pass-through: `body` runs with the
4755/// full budget and nothing is leased.
4756///
4757/// A lease that cannot be acquired within `budget` is reported as a single
4758/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
4759/// verification — the same shape a spawn failure already takes in
4760/// [`run_commands`], so a caller need not special-case it.
4761#[allow(clippy::too_many_arguments)]
4762async fn with_cache_lease<'s, F, Fut>(
4763    state: &'s mut RunState,
4764    cache_dir: Option<&Path>,
4765    node: &str,
4766    seat: &str,
4767    worktree: &Path,
4768    head: &str,
4769    budget: Duration,
4770    context: &str,
4771    body: F,
4772) -> (Vec<CommandOutcome>, bool)
4773where
4774    F: FnOnce(&'s mut RunState, Duration) -> Fut,
4775    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
4776{
4777    let Some(cache_dir) = cache_dir else {
4778        let (outcomes, retried, _timed_out_pids) = body(state, budget).await;
4779        return (outcomes, retried);
4780    };
4781    let home = crate::run::home();
4782    let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
4783    let started = Instant::now();
4784    let guard = match acquire_cache_lease(state, cache_dir, &owner, budget, context).await {
4785        Ok(g) => g,
4786        Err(e) => {
4787            return (
4788                vec![CommandOutcome {
4789                    command: "(waiting for the shared build cache)".to_owned(),
4790                    code: None,
4791                    output_tail: e.to_string(),
4792                    duration_ms: started.elapsed().as_millis() as u64,
4793                    resource_blocked: true,
4794                }],
4795                false,
4796            );
4797        }
4798    };
4799    let identity = crate::cache::Identity::new(worktree, head);
4800    if let Err(e) = crate::cache::ensure_fresh(&home, cache_dir, &identity) {
4801        // A failed freshness check means this process cannot vouch for what
4802        // is sitting in the cache right now - on Windows this is exactly the
4803        // "a stale test executable is still locked, `cargo clean -p` cannot
4804        // remove it" case the evidence log records. Running verify anyway
4805        // and reporting whatever it says would let a result nobody can trust
4806        // stand for the tree it claims to have checked; fail the step
4807        // instead of the patch.
4808        state.event(
4809            "verify",
4810            format!(
4811                "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
4812                worktree.display(),
4813                short(head)
4814            ),
4815        );
4816        guard.release();
4817        return (
4818            vec![CommandOutcome {
4819                command: "(confirming the shared build cache is fresh)".to_owned(),
4820                code: None,
4821                output_tail: e.to_string(),
4822                duration_ms: started.elapsed().as_millis() as u64,
4823                resource_blocked: true,
4824            }],
4825            false,
4826        );
4827    }
4828    let remaining = budget.saturating_sub(started.elapsed());
4829    let (outcomes, retried, timed_out_pids) = body(state, remaining).await;
4830    // A timed-out command's process was only *asked* to die (`kill_on_drop`,
4831    // `start_kill`); confirm it actually has before handing the directory to
4832    // the next acquirer. See `wait_for_timed_out_children_to_die`'s own doc
4833    // for what this can and cannot see.
4834    if !timed_out_pids.is_empty() {
4835        wait_for_timed_out_children_to_die(&timed_out_pids).await;
4836    }
4837    guard.release();
4838    (outcomes, retried)
4839}
4840
4841/// Poll `pids` — commands [`run_commands`] reports as still running when its
4842/// own timeout elapsed — until every one is confirmed gone, or
4843/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
4844///
4845/// Real confirmation where confirmation is possible, not a substitute for
4846/// full process-tree observation: a grandchild the timed-out process spawned
4847/// and that survives independently of it is invisible to a pid check the
4848/// same way it always was, and continuing to observe and collect *that*
4849/// stays a different piece of work with its own owner. This only narrows a
4850/// fixed blind wait into an actual check of the pids this process does know
4851/// about.
4852async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
4853    wait_for_pids_with(
4854        pids,
4855        crate::proc::pid_alive,
4856        LEASE_RELEASE_POLL,
4857        LEASE_RELEASE_MAX_WAIT,
4858    )
4859    .await;
4860}
4861
4862/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
4863/// interval and ceiling supplied by the caller, so the polling *logic* -
4864/// returns as soon as every pid reports dead, gives up at the ceiling
4865/// otherwise - is testable on millisecond durations without asking the real
4866/// OS about a pid at all.
4867async fn wait_for_pids_with<F: Fn(u32) -> bool>(
4868    pids: &[u32],
4869    alive: F,
4870    poll: Duration,
4871    max_wait: Duration,
4872) {
4873    let deadline = Instant::now() + max_wait;
4874    loop {
4875        if pids.iter().all(|&pid| !alive(pid)) {
4876            return;
4877        }
4878        if Instant::now() >= deadline {
4879            return;
4880        }
4881        tokio::time::sleep(poll).await;
4882    }
4883}
4884
4885/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
4886/// admission that it could not even get a verify command to run, as opposed
4887/// to evidence the command actually produced? A caller that would otherwise
4888/// read a resource-blocked outcome as a red command must check this first:
4889/// see [`Runner::gate`], which retries rather than records `Blocked` when
4890/// this is true.
4891fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
4892    outcomes.iter().any(|o| o.resource_blocked)
4893}
4894
4895/// Describe one verify command's outcome for the event log, distinguishing a
4896/// build/link failure — the toolchain never produced a binary to run — from
4897/// an actual test failure, since only the latter is a verdict on the patch.
4898fn e2e_outcome_label(o: &CommandOutcome) -> String {
4899    if o.ok() {
4900        return "pass".to_owned();
4901    }
4902    let reason = if o.build_failed() {
4903        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
4904    } else {
4905        format!("FAIL ({:?})", o.code)
4906    };
4907    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
4908}
4909
4910/// Run `verify.e2e`, retrying once if the first attempt could not build or
4911/// link — a build/link failure is frequently a race against a shared
4912/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
4913/// `verify` event per command, tagged with `context` (normally `"round N"`)
4914/// so the two call sites that need this — the ordinary per-round leg in
4915/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
4916/// it will ever call a round green — read identically in the event log.
4917async fn run_e2e_with_retry(
4918    state: &mut RunState,
4919    shell: &[String],
4920    commands: &[String],
4921    worktree: &Path,
4922    timeout: Duration,
4923    context: &str,
4924) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
4925    let (mut e2e, mut timed_out_pids) = run_commands(shell, commands, worktree, timeout).await;
4926    for o in &e2e {
4927        state.event(
4928            "verify",
4929            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
4930        );
4931    }
4932    // A build/link failure is not a verdict on the patch — it is frequently a
4933    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
4934    // one retry before letting a red like that decide the round.
4935    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
4936    if verify_retried {
4937        state.event(
4938            "verify",
4939            format!(
4940                "{context}: verify could not build/link, not a test result — retrying once \
4941                 before concluding"
4942            ),
4943        );
4944        let retried = run_commands(shell, commands, worktree, timeout).await;
4945        e2e = retried.0;
4946        // Both attempts' timeouts matter, not just the last one: the first
4947        // attempt's descendants may still be alive alongside the retry's.
4948        timed_out_pids.extend(retried.1);
4949        for o in &e2e {
4950            state.event(
4951                "verify",
4952                format!(
4953                    "{context}: retry `{}` -> {}",
4954                    o.command,
4955                    e2e_outcome_label(o)
4956                ),
4957            );
4958        }
4959    }
4960    (e2e, verify_retried, timed_out_pids)
4961}
4962
4963/// Run configured shell commands in `cwd`, in order. The second element is
4964/// the pid of every command that hit `timeout` and was still running when
4965/// this stopped waiting on it (best-effort: `None` when the platform did not
4966/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
4967/// that releases a shared resource afterward needs to know.
4968async fn run_commands(
4969    shell: &[String],
4970    commands: &[String],
4971    cwd: &Path,
4972    timeout: Duration,
4973) -> (Vec<CommandOutcome>, Vec<u32>) {
4974    let mut out = Vec::new();
4975    let mut timed_out_pids = Vec::new();
4976    for command in commands {
4977        let started = Instant::now();
4978        let mut cmd = tokio::process::Command::new(&shell[0]);
4979        cmd.quiet();
4980        cmd.args(&shell[1..])
4981            .arg(command)
4982            .current_dir(cwd)
4983            .stdin(std::process::Stdio::null())
4984            .stdout(std::process::Stdio::piped())
4985            .stderr(std::process::Stdio::piped())
4986            .kill_on_drop(true);
4987        let spawned = cmd.spawn();
4988        let (code, body) = match spawned {
4989            Ok(child) => {
4990                // Captured before the child is consumed below: `kill_on_drop`
4991                // only *asks* the process to die when the timeout branch
4992                // drops it, and the pid is the only way anyone downstream can
4993                // later check whether that request actually took.
4994                let pid = child.id();
4995                match tokio::time::timeout(timeout, child.wait_with_output()).await {
4996                    Ok(Ok(o)) => {
4997                        let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
4998                        body.push_str(&String::from_utf8_lossy(&o.stderr));
4999                        (o.status.code(), body)
5000                    }
5001                    Ok(Err(e)) => (None, format!("failed to run: {e}")),
5002                    Err(_) => {
5003                        if let Some(pid) = pid {
5004                            timed_out_pids.push(pid);
5005                        }
5006                        (None, format!("timed out after {}s", timeout.as_secs()))
5007                    }
5008                }
5009            }
5010            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
5011        };
5012        out.push(CommandOutcome {
5013            command: command.clone(),
5014            code,
5015            output_tail: tail(&body, OUTPUT_TAIL),
5016            duration_ms: started.elapsed().as_millis() as u64,
5017            resource_blocked: false,
5018        });
5019    }
5020    (out, timed_out_pids)
5021}
5022
5023/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
5024/// section (`report::run`) and in the `merge` event this node records — for
5025/// the operator to run by hand.
5026///
5027/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
5028/// branch whose ruleset forbids merge commits (GitHub's "must not contain
5029/// merge commits", or "require linear history") rejects the push a `--no-ff`
5030/// merge would produce, which is exactly the guidance this function replaces.
5031/// `message`'s first line becomes the squash commit's subject, matching the
5032/// note `report::run` prints alongside this command — see that function for
5033/// why an explicit subject is not optional there.
5034fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
5035    let repo = repo.display();
5036    match style {
5037        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
5038        MergeStyle::Squash => {
5039            let subject = message.lines().next().unwrap_or(branch);
5040            format!(
5041                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
5042            )
5043        }
5044        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
5045    }
5046}
5047
5048/// The merge commit / pull request body: the task, and — when the winning
5049/// review round was not clean — the findings still open and whatever the
5050/// fixer declined, so `merge = "pr"` hands the reader the same material
5051/// `magi show` does rather than a pull request that reads clean while
5052/// `run.json` disagrees.
5053///
5054/// The first line doubles as the pull request title (`gh_pr_create`) and the
5055/// squash/merge commit subject (`manual_merge_command`), both of which take
5056/// it via `message.lines().next()` rather than as a separate argument — so it
5057/// has to be the task's own opening line, not run/candidate bookkeeping.
5058/// "Merge magi run ec12 (candidate B)" told a reader nothing about what
5059/// landed once the run id had scrolled off the PR list. That bookkeeping
5060/// still needs to be findable, just not from the title: the branch name
5061/// already carries it (`RunState::branch_for`), and the footer below repeats
5062/// it as plain tags for a reader holding only the merged commit or the PR
5063/// body.
5064///
5065/// `state.instruction` can open with blank lines — a `--file` task is passed
5066/// through verbatim (`task_text` only rejects a body that is blank
5067/// *entirely*) — and `.lines().next()` on those reads back as `Some("")`, not
5068/// `None`, so `gh_pr_create`'s `unwrap_or("magi run")` never fires and `gh pr
5069/// create` would be asked for an empty `--title`. `trim_start` drops exactly
5070/// those leading blank lines so the first line is the task's real opening
5071/// line, and the empty-after-trim case (a whitespace-only instruction) falls
5072/// back the same way `queue::title_from` does for the same situation.
5073fn pr_body(state: &RunState, winner: char) -> String {
5074    let instruction = state.instruction.trim_start();
5075    let mut message = if instruction.is_empty() {
5076        "(empty task)".to_owned()
5077    } else {
5078        instruction.to_owned()
5079    };
5080
5081    let open = state.open_findings();
5082    if !open.is_empty() {
5083        message.push_str("\n\n## Open review findings\n\n");
5084        for f in &open {
5085            message.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
5086        }
5087    }
5088
5089    if let Some(fix) = state.reviews.last().and_then(|r| r.fix.as_ref())
5090        && !fix.rejected.is_empty()
5091    {
5092        message.push_str("\n## Declined by the fixer\n\n");
5093        for r in &fix.rejected {
5094            message.push_str(&format!("- `{}`: {}\n", r.id, r.why));
5095        }
5096    }
5097
5098    message.push_str(&format!(
5099        "\n\n---\nmagi:run/{} magi:candidate-{}\n",
5100        state.id,
5101        winner.to_ascii_lowercase()
5102    ));
5103
5104    message
5105}
5106
5107/// `gh pr create`, returning the PR url.
5108async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
5109    let title = body.lines().next().unwrap_or("magi run").to_owned();
5110    let out = tokio::process::Command::new("gh")
5111        .args([
5112            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
5113        ])
5114        .current_dir(cwd)
5115        .quiet()
5116        .stdin(std::process::Stdio::null())
5117        .output()
5118        .await
5119        .context("spawn gh")?;
5120    if out.status.success() {
5121        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
5122    } else {
5123        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
5124    }
5125}
5126
5127/// Tear a run's worktrees and branches down.
5128pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
5129    let repo = state.repo.clone();
5130    let root = state.worktree_root();
5131    let winner = state.tally.as_ref().map(|t| t.winner);
5132    let mut removed = Vec::new();
5133
5134    for i in 0..state.candidates.len() {
5135        let c = state.candidates[i].clone();
5136        let is_winner = Some(c.label) == winner;
5137        if is_winner && !drop_winner {
5138            continue;
5139        }
5140        if c.worktree.exists() {
5141            git::worktree_remove(&repo, &c.worktree).await.ok();
5142            removed.push(c.worktree.to_string_lossy().into_owned());
5143        }
5144        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
5145            git::branch_delete(&repo, &c.branch).await.ok();
5146            removed.push(c.branch.clone());
5147        }
5148        state.candidates[i].folded = true;
5149    }
5150
5151    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
5152        let path = name.path();
5153        let keep = !drop_winner
5154            && winner.is_some_and(|w| {
5155                path.file_name()
5156                    .is_some_and(|n| n == format!("cand-{w}").as_str())
5157            });
5158        if keep {
5159            continue;
5160        }
5161        git::worktree_remove(&repo, &path).await.ok();
5162        removed.push(path.to_string_lossy().into_owned());
5163    }
5164
5165    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
5166    // judge worktrees, so once the loop above has cleared all of them out,
5167    // the parent is a bare directory nobody else was ever going to remove -
5168    // git only ever managed what was inside it. Left alone, one of these
5169    // accumulates per fully-folded run; the operator's own machine had 74.
5170    // `remove_if_empty` re-checks rather than assuming: a run whose winner
5171    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
5172    // so does anything a run never claimed that happens to share the bay.
5173    remove_if_empty(&root);
5174
5175    if state.enabled_worktree_config && drop_winner {
5176        // A release, not a raw disable: some sibling run in this repository
5177        // may still hold its own reference (see `git::acquire_worktree_config`),
5178        // and only the last release actually turns the setting back off.
5179        git::release_worktree_config(&repo).await.ok();
5180        state.enabled_worktree_config = false;
5181    }
5182    state.save()?;
5183    Ok(removed)
5184}
5185
5186/// Remove `dir` if it exists and has nothing in it.
5187///
5188/// Best-effort and silent by design: a directory that is not empty (a run
5189/// whose winner is still parked there, a stray file some other process left)
5190/// is exactly the case this must refuse, and a directory that is already gone
5191/// is not a failure worth reporting either. `std::fs::remove_dir` itself
5192/// already refuses a non-empty directory, so the emptiness check below is
5193/// belt, not suspenders - it is what keeps this from ever attempting the
5194/// removal in the case that matters, rather than trusting `remove_dir`'s
5195/// error path to have no side effects if it ever changed.
5196fn remove_if_empty(dir: &Path) {
5197    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
5198        std::fs::remove_dir(dir).ok();
5199    }
5200}
5201
5202/// Severity of the worst open finding in the last review round, for reporting.
5203pub fn worst_open(state: &RunState) -> Option<Severity> {
5204    state
5205        .reviews
5206        .last()?
5207        .reviews
5208        .iter()
5209        .flat_map(|r| r.findings.iter())
5210        .map(|f| f.severity)
5211        .max()
5212}
5213
5214#[cfg(test)]
5215mod tests {
5216    use super::*;
5217    use crate::run::GateStatus;
5218    use std::collections::BTreeMap;
5219    use std::time::Duration;
5220
5221    fn conductor() -> AgentSpec {
5222        AgentSpec {
5223            id: "conductor".to_owned(),
5224            kind: crate::config::AgentKind::Command,
5225            model: None,
5226            command: vec!["true".to_owned()],
5227            extra_args: Vec::new(),
5228            env: BTreeMap::new(),
5229            prompt_delivery: None,
5230        }
5231    }
5232
5233    #[test]
5234    fn remove_if_empty_only_ever_takes_a_bare_directory() {
5235        let dir = tempfile::tempdir().unwrap();
5236        let bay = dir.path().join("ffff");
5237
5238        // Not there yet: nothing to do, nothing to panic on.
5239        remove_if_empty(&bay);
5240        assert!(!bay.exists());
5241
5242        // Something still inside - the winner's worktree, or a stray file -
5243        // keeps the directory standing.
5244        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
5245        remove_if_empty(&bay);
5246        assert!(bay.exists(), "non-empty directory must survive");
5247
5248        // Once the last entry is gone, so is the directory itself.
5249        std::fs::remove_dir(bay.join("cand-A")).unwrap();
5250        remove_if_empty(&bay);
5251        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
5252    }
5253
5254    // `round_is_clean` is the exact decision this task fixed: a round with a
5255    // seat that never answered must not read the same as a round every seat
5256    // actually reviewed. These are deterministic and process-free by design —
5257    // the equivalent end-to-end check (a real reviewer timing out under a
5258    // live graph run) is a genuine race against wall-clock contention, and a
5259    // spawn slow enough to blow even a generous budget under a loaded test
5260    // run must not turn this specific regression check flaky.
5261
5262    #[test]
5263    fn a_full_panel_that_found_nothing_is_clean() {
5264        assert!(round_is_clean(
5265            0,
5266            true,
5267            2,
5268            2,
5269            0,
5270            IncompleteReviewPolicy::Block
5271        ));
5272    }
5273
5274    #[test]
5275    fn a_missing_seat_is_never_clean_under_the_default_policy() {
5276        assert!(!round_is_clean(
5277            0,
5278            true,
5279            1,
5280            2,
5281            0,
5282            IncompleteReviewPolicy::Block
5283        ));
5284    }
5285
5286    #[test]
5287    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
5288        assert!(!round_is_clean(
5289            1,
5290            true,
5291            1,
5292            2,
5293            0,
5294            IncompleteReviewPolicy::Warn
5295        ));
5296    }
5297
5298    #[test]
5299    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
5300        assert!(round_is_clean(
5301            0,
5302            true,
5303            1,
5304            2,
5305            0,
5306            IncompleteReviewPolicy::Warn
5307        ));
5308    }
5309
5310    #[test]
5311    fn a_full_panel_with_an_open_finding_is_not_clean() {
5312        assert!(!round_is_clean(
5313            1,
5314            true,
5315            2,
5316            2,
5317            0,
5318            IncompleteReviewPolicy::Block
5319        ));
5320    }
5321
5322    #[test]
5323    fn a_full_panel_with_a_red_e2e_is_not_clean() {
5324        assert!(!round_is_clean(
5325            0,
5326            false,
5327            2,
5328            2,
5329            0,
5330            IncompleteReviewPolicy::Block
5331        ));
5332    }
5333
5334    // The stall this task closes: under the default `block` policy, a seat
5335    // missing only because it was rate limited must not force a wait for a
5336    // session limit that will not lift by the next round. `round_is_clean`
5337    // is where that quorum carve-out lives; the review loop around it never
5338    // changes what a reviewer's vote or a finding's severity means.
5339
5340    #[test]
5341    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
5342        // 1 of 2 answered, and the one missing was quota'd — the exact
5343        // "review-2 rate limited (quota)" shape from the field report.
5344        assert!(round_is_clean(
5345            0,
5346            true,
5347            1,
5348            2,
5349            1,
5350            IncompleteReviewPolicy::Block
5351        ));
5352    }
5353
5354    #[test]
5355    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
5356        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
5357        // not a quota loss (`quota_missing` stays 0) — worth another try.
5358        assert!(!round_is_clean(
5359            0,
5360            true,
5361            1,
5362            2,
5363            0,
5364            IncompleteReviewPolicy::Block
5365        ));
5366    }
5367
5368    #[test]
5369    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
5370        assert!(!round_is_clean(
5371            1,
5372            true,
5373            1,
5374            2,
5375            1,
5376            IncompleteReviewPolicy::Block
5377        ));
5378        assert!(!round_is_clean(
5379            0,
5380            false,
5381            1,
5382            2,
5383            1,
5384            IncompleteReviewPolicy::Block
5385        ));
5386    }
5387
5388    #[test]
5389    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
5390        // Every seat quota'd, nobody answered: there is no panel to decide
5391        // on, so this must fall through to the existing block-and-retry
5392        // fallback rather than call an unreviewed patch clean.
5393        assert!(!round_is_clean(
5394            0,
5395            true,
5396            0,
5397            2,
5398            2,
5399            IncompleteReviewPolicy::Block
5400        ));
5401    }
5402
5403    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
5404        CommandOutcome {
5405            command: "test".to_owned(),
5406            code,
5407            output_tail: String::new(),
5408            duration_ms: 0,
5409            resource_blocked,
5410        }
5411    }
5412
5413    #[test]
5414    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
5415        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
5416        assert!(
5417            !verify_inconclusive(&[outcome(Some(1), false)]),
5418            "an ordinary failure is still evidence about the patch"
5419        );
5420        assert!(verify_inconclusive(&[outcome(None, true)]));
5421        assert!(
5422            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
5423            "one inconclusive outcome taints the whole batch"
5424        );
5425        assert!(!verify_inconclusive(&[]));
5426    }
5427
5428    #[tokio::test]
5429    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
5430        // Alive for the first two checks, then dead - confirms the loop
5431        // actually re-polls rather than deciding once and sleeping out the
5432        // ceiling regardless.
5433        let calls = std::sync::atomic::AtomicUsize::new(0);
5434        let started = Instant::now();
5435        wait_for_pids_with(
5436            &[123],
5437            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
5438            Duration::from_millis(5),
5439            Duration::from_secs(5),
5440        )
5441        .await;
5442        assert!(
5443            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
5444            "must keep checking rather than deciding on the first answer"
5445        );
5446        assert!(
5447            started.elapsed() < Duration::from_secs(1),
5448            "must return the moment it is confirmed dead, not wait out the ceiling"
5449        );
5450    }
5451
5452    #[tokio::test]
5453    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
5454        let started = Instant::now();
5455        wait_for_pids_with(
5456            &[123],
5457            |_| true, // never reports dead
5458            Duration::from_millis(5),
5459            Duration::from_millis(30),
5460        )
5461        .await;
5462        let elapsed = started.elapsed();
5463        assert!(
5464            elapsed >= Duration::from_millis(30),
5465            "must not give up before its own ceiling: {elapsed:?}"
5466        );
5467        assert!(
5468            elapsed < Duration::from_secs(1),
5469            "must not wait past its own ceiling either: {elapsed:?}"
5470        );
5471    }
5472
5473    #[tokio::test]
5474    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
5475        let started = Instant::now();
5476        wait_for_pids_with(
5477            &[],
5478            |_| true,
5479            Duration::from_secs(5),
5480            Duration::from_secs(5),
5481        )
5482        .await;
5483        assert!(
5484            started.elapsed() < Duration::from_millis(200),
5485            "an empty pid list has nothing to confirm"
5486        );
5487    }
5488
5489    // `review_conclusion` is the exact decision the review hand-off task
5490    // fixed: a round budget spent (or a tree that stopped moving) must not
5491    // collapse into `Blocked` regardless of what verification actually
5492    // said. Deterministic and process-free for the same reason the
5493    // `round_is_clean` family above is.
5494    fn review_round(
5495        clean: bool,
5496        blocking: usize,
5497        answered: usize,
5498        expected: usize,
5499        progressed: bool,
5500        e2e_ok: bool,
5501    ) -> ReviewRound {
5502        ReviewRound {
5503            round: 1,
5504            head: "h".to_owned(),
5505            verified_head: None,
5506            reviews: Vec::new(),
5507            e2e: vec![CommandOutcome {
5508                command: "test".to_owned(),
5509                code: Some(if e2e_ok { 0 } else { 1 }),
5510                output_tail: String::new(),
5511                duration_ms: 0,
5512                resource_blocked: false,
5513            }],
5514            verify_retried: false,
5515            e2e_deferred: false,
5516            e2e_defer_reason: None,
5517            fix: None,
5518            blocking,
5519            answered,
5520            expected,
5521            clean,
5522            progressed,
5523            vote_split: false,
5524            reconsideration: Vec::new(),
5525            verdict: None,
5526        }
5527    }
5528
5529    #[test]
5530    fn review_conclusion_is_none_when_nothing_has_run() {
5531        assert_eq!(review_conclusion(&[], 3), None);
5532    }
5533
5534    #[test]
5535    fn review_conclusion_is_none_while_rounds_remain() {
5536        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
5537        assert_eq!(review_conclusion(&rounds, 3), None);
5538    }
5539
5540    #[test]
5541    fn review_conclusion_is_gating_once_a_round_is_clean() {
5542        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
5543        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
5544    }
5545
5546    #[test]
5547    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
5548        let rounds = vec![
5549            review_round(false, 1, 2, 2, true, true),
5550            review_round(false, 1, 2, 2, true, true),
5551        ];
5552        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
5553    }
5554
5555    #[test]
5556    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
5557        let rounds = vec![
5558            review_round(false, 1, 2, 2, true, true),
5559            review_round(false, 1, 2, 2, true, false),
5560        ];
5561        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
5562    }
5563
5564    #[test]
5565    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
5566        // Missing input, not a verified tree — never a hand-off candidate.
5567        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
5568        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
5569    }
5570
5571    #[test]
5572    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
5573        let rounds = vec![
5574            review_round(false, 1, 2, 2, false, true),
5575            review_round(false, 1, 2, 2, false, true),
5576        ];
5577        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
5578    }
5579
5580    fn secs(n: u64) -> Duration {
5581        Duration::from_secs(n)
5582    }
5583
5584    /// A throwaway repo with one commit on `main`, for tests that need `merge`
5585    /// to make real (and, if it runs at all, real*ly fail*) git calls.
5586    fn init_repo(dir: &Path) {
5587        let run = |args: &[&str]| {
5588            let out = std::process::Command::new("git")
5589                .args(args)
5590                .current_dir(dir)
5591                .quiet()
5592                .output()
5593                .expect("spawn git");
5594            assert!(
5595                out.status.success(),
5596                "git {args:?} failed: {}",
5597                String::from_utf8_lossy(&out.stderr)
5598            );
5599        };
5600        run(&["init", "-b", "main"]);
5601        run(&["config", "user.name", "magi test"]);
5602        run(&["config", "user.email", "magi@example.com"]);
5603        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
5604        run(&["add", "-A"]);
5605        run(&["commit", "-m", "init"]);
5606    }
5607
5608    // `settle_questions` is what closes the ghost the phone showed: a run's
5609    // seat asked something, the run then ended, and nothing was left to
5610    // abandon the question it left `open`. `HOME` is a process-wide
5611    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
5612    // first time it runs in the binary — every test below still reaches the
5613    // same directory whichever call won, and each gets its own run id from
5614    // `RunState::new`, so they never collide there.
5615    fn ask_test_home() {
5616        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
5617    }
5618
5619    /// A minimal, git-free `Runner` at a given status — `settle_questions`
5620    /// reads nothing else off it.
5621    fn runner_at(status: RunStatus) -> Runner {
5622        let mut state = RunState::new(
5623            PathBuf::from("/nonexistent/repo"),
5624            "main".to_owned(),
5625            "deadbeef".to_owned(),
5626            "task".to_owned(),
5627            Config::default(),
5628        );
5629        state.status = status;
5630        Runner {
5631            state,
5632            roles: ResolvedRoles {
5633                implementers: Vec::new(),
5634                judges: Vec::new(),
5635                reviewers: Vec::new(),
5636                fixer: None,
5637                conductor: conductor(),
5638            },
5639            sem: Arc::new(Semaphore::new(1)),
5640            pause: Pause::new(),
5641            interrupt: Pause::new(),
5642        }
5643    }
5644
5645    /// `park_here` folding in the reason `Pause::park_because` recorded -
5646    /// this is what lets an operator reading a run's events tell an
5647    /// interrupt-driven park from an ordinary shutdown park.
5648    #[test]
5649    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
5650        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5651        let mut runner = runner_at(RunStatus::Implementing);
5652        let interrupt = Pause::new();
5653        runner.watch_interrupt(interrupt.clone());
5654
5655        interrupt.park_because("task a1b2 asked to run first");
5656
5657        assert!(runner.park_here().expect("park_here"));
5658        assert!(runner.state.parked);
5659        let last = runner.state.events.last().expect("a park event");
5660        assert_eq!(last.node, "park");
5661        assert!(
5662            last.message.contains("task a1b2 asked to run first"),
5663            "expected the interrupt reason in {:?}",
5664            last.message
5665        );
5666    }
5667
5668    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
5669    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
5670    /// cleared) must not make a *different* run - one only watching its own,
5671    /// unshared interrupt `Pause` - see itself as parked. If a future change
5672    /// ever collapsed these back into one handle, the interrupt scheduler
5673    /// would park every run for the rest of the daemon's life, not just the
5674    /// one it meant to interrupt.
5675    #[test]
5676    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
5677        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5678        let mut runner = runner_at(RunStatus::Implementing);
5679        let shutdown = Pause::new();
5680        runner.on_pause(shutdown.clone());
5681        let interrupt = Pause::new();
5682        runner.watch_interrupt(interrupt.clone());
5683
5684        // Nobody has asked for anything yet.
5685        assert!(!runner.park_here().expect("park_here"));
5686        assert!(!runner.state.parked);
5687
5688        // Only the interrupt handle fires; the shutdown handle stays clear.
5689        interrupt.park_because("test");
5690        assert!(!shutdown.parked());
5691        assert!(runner.park_here().expect("park_here"));
5692    }
5693
5694    /// The property every prior attempt at this feature failed to pin down:
5695    /// asking a run to park while one of its nodes has a real, in-flight
5696    /// async operation running (an agent call, in production) must not cut
5697    /// that operation short. `park_here` is only ever consulted *between*
5698    /// `execute`'s node calls - see its own doc - so nothing inside a node
5699    /// can observe a park request until the node itself returns. This proves
5700    /// that structurally, with real `tokio` concurrency and a channel
5701    /// handshake (never a sleep, which would only prove "usually", not
5702    /// "cannot"): the "node" below reports that it has genuinely started,
5703    /// and only then is the park requested; the node still has to be told to
5704    /// finish before `park_here` is ever called, exactly mirroring every
5705    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
5706    /// in `execute`.
5707    #[tokio::test]
5708    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
5709        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5710        let mut runner = runner_at(RunStatus::Implementing);
5711        let interrupt = Pause::new();
5712        runner.watch_interrupt(interrupt.clone());
5713
5714        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
5715        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
5716
5717        // Stands in for one node's in-flight agent call: it proves it has
5718        // genuinely started, then blocks - exactly as a spawned CLI process
5719        // does - until told to finish.
5720        let node = async move {
5721            started_tx.send(()).expect("send started");
5722            finish_rx.await.expect("recv finish");
5723            "node finished"
5724        };
5725
5726        let interrupter = async move {
5727            started_rx.await.expect("recv started");
5728            // The call is now genuinely in flight. Ask it to park.
5729            interrupt.park_because("higher-priority task waiting");
5730            // Nothing the node does can observe this yet - there is no
5731            // check inside it, by construction - so let the executor run
5732            // anything pending and then let the node finish on its own.
5733            tokio::task::yield_now().await;
5734            finish_tx.send(()).expect("send finish");
5735        };
5736
5737        let (node_result, ()) = tokio::join!(node, interrupter);
5738        assert_eq!(
5739            node_result, "node finished",
5740            "the in-flight call ran to completion"
5741        );
5742
5743        // Only now, at the boundary the real `execute` would check right
5744        // after this node, does the park take effect.
5745        assert!(runner.park_here().expect("park_here"));
5746        assert!(runner.state.parked);
5747    }
5748
5749    /// A run parked mid-competition carries every field it had accumulated
5750    /// through the exact same disk round-trip an ordinary resume uses -
5751    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
5752    /// Nothing about parking for an interrupt is a special case of that path;
5753    /// this is what proves it rather than assuming it.
5754    #[test]
5755    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
5756        crate::run::set_home(std::env::temp_dir().join("magi-graph-interrupt-tests-home"));
5757        let mut runner = runner_at(RunStatus::Judging);
5758        // `Runner::resume` re-resolves roles from the saved config, which
5759        // refuses an empty roster - give it the same minimal one `conductor`
5760        // itself uses.
5761        runner.state.config.agents = vec![conductor()];
5762        runner.state.candidates = vec![Candidate {
5763            index: 0,
5764            label: 'A',
5765            agent: "alpha".to_owned(),
5766            branch: "magi/x/A".to_owned(),
5767            worktree: PathBuf::from("/nonexistent/worktree"),
5768            summary: "did the thing".to_owned(),
5769            stat: "1 file changed".to_owned(),
5770            files: 1,
5771            commits: 1,
5772            empty: false,
5773            failed: None,
5774            duration_ms: 1234,
5775            folded: false,
5776        }];
5777        let run_id = runner.state.id.clone();
5778
5779        let interrupt = Pause::new();
5780        runner.watch_interrupt(interrupt.clone());
5781        interrupt.park_because("task c3d4 asked to run first");
5782        assert!(runner.park_here().expect("park_here"));
5783
5784        let resumed = Runner::resume(&run_id).expect("resume");
5785        assert_eq!(resumed.state.candidates.len(), 1);
5786        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
5787        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
5788        assert_eq!(resumed.state.status, runner.state.status);
5789        assert!(
5790            resumed.state.parked,
5791            "still parked until `execute` actually walks the graph again"
5792        );
5793        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
5794    }
5795
5796    /// A fresh open question on `run`, stored and handed back for assertions.
5797    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
5798        let mut q = ask::Question::new(
5799            run.to_owned(),
5800            "implement".to_owned(),
5801            "impl-A".to_owned(),
5802            "Which storage backend should the cache use?".to_owned(),
5803            String::new(),
5804            vec!["SQLite".to_owned(), "Redis".to_owned()],
5805        );
5806        store.put(&mut q).unwrap();
5807        q
5808    }
5809
5810    #[test]
5811    fn a_failed_runs_open_question_is_abandoned() {
5812        ask_test_home();
5813        let store = ask::Questions::open();
5814        let mut runner = runner_at(RunStatus::Failed);
5815        let run = runner.state.id.clone();
5816        let q = ask_open_question(&store, &run);
5817
5818        runner.settle_questions();
5819
5820        let back = store.get(&q.id).unwrap();
5821        assert!(
5822            !back.status.open(),
5823            "the seat that asked died with the run; nobody is left to read an answer"
5824        );
5825        assert!(
5826            back.detail.contains(&run) && back.detail.contains("failed"),
5827            "the reason names what the run became, not just that it is gone: {}",
5828            back.detail
5829        );
5830    }
5831
5832    #[test]
5833    fn a_merged_runs_open_question_is_abandoned_too() {
5834        ask_test_home();
5835        let store = ask::Questions::open();
5836        // A run that finishes cleanly still leaves nobody to read an answer -
5837        // this is not only a failure-path cleanup.
5838        for status in [RunStatus::Merged, RunStatus::Ready] {
5839            let mut runner = runner_at(status);
5840            let run = runner.state.id.clone();
5841            let q = ask_open_question(&store, &run);
5842
5843            runner.settle_questions();
5844
5845            let back = store.get(&q.id).unwrap();
5846            assert!(
5847                !back.status.open(),
5848                "{status:?} run's question must not outlive the run"
5849            );
5850        }
5851    }
5852
5853    #[test]
5854    fn a_still_resumable_runs_open_question_is_left_alone() {
5855        ask_test_home();
5856        let store = ask::Questions::open();
5857        // `Blocked` and `Stalled` can still be resumed — the candidates, the
5858        // review round and the seat sessions are all still on disk — so a
5859        // question asked mid-round may yet get a real answer from a real
5860        // resume. Sweeping it here would be exactly the failure mode this
5861        // whole feature exists to avoid on the other side.
5862        for status in [RunStatus::Blocked, RunStatus::Stalled] {
5863            let mut runner = runner_at(status);
5864            let run = runner.state.id.clone();
5865            let q = ask_open_question(&store, &run);
5866
5867            runner.settle_questions();
5868
5869            let back = store.get(&q.id).unwrap();
5870            assert!(
5871                back.status.open(),
5872                "{status:?} is still alive; the question must still be waiting"
5873            );
5874        }
5875    }
5876
5877    #[test]
5878    fn settle_questions_never_touches_an_already_answered_question() {
5879        ask_test_home();
5880        let store = ask::Questions::open();
5881        let mut runner = runner_at(RunStatus::Failed);
5882        let run = runner.state.id.clone();
5883        let mut q = ask_open_question(&store, &run);
5884        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
5885            .unwrap();
5886        store.put(&mut q).unwrap();
5887
5888        // Called twice, the way a crash-recovered daemon reclaim and the
5889        // graph's own cleanup both can for the same run — `abandon_for_run`
5890        // only ever touches what is still open, so this must be inert both
5891        // times, not merely the second.
5892        runner.settle_questions();
5893        runner.settle_questions();
5894
5895        let back = store.get(&q.id).unwrap();
5896        assert_eq!(
5897            back.status,
5898            ask::QuestionStatus::Answered,
5899            "a real answer is a decision on record, never overwritten by a sweep"
5900        );
5901    }
5902
5903    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
5904    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
5905    /// without merging, whose winner is still the operator's answer to read.
5906    /// Nothing previously called `fold_run` itself with a real `tally`, so
5907    /// this is the first test to pin down the one distinction the whole
5908    /// automatic-fold feature depends on: the winner's worktree and branch
5909    /// must survive, everything else sharing the run's worktree bay - a
5910    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
5911    /// second sweep treats every non-winner directory under the bay alike -
5912    /// must not.
5913    #[tokio::test]
5914    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
5915        crate::run::set_home(std::env::temp_dir().join("magi-graph-fold-run-tests-home"));
5916        let tmp = tempfile::tempdir().expect("tempdir");
5917        let repo = tmp.path().join("repo");
5918        std::fs::create_dir_all(&repo).unwrap();
5919        init_repo(&repo);
5920
5921        let mut config = Config::default();
5922        config.graph.worktree_root = Some(tmp.path().join("wt"));
5923
5924        let mut state = RunState::new(
5925            repo.clone(),
5926            "main".to_owned(),
5927            "deadbeef".to_owned(),
5928            "task".to_owned(),
5929            config,
5930        );
5931        let root = state.worktree_root();
5932        let wt_a = root.join("cand-A");
5933        let wt_b = root.join("cand-B");
5934        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
5935            .await
5936            .expect("worktree A");
5937        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
5938            .await
5939            .expect("worktree B");
5940
5941        state.candidates = vec![
5942            Candidate {
5943                index: 0,
5944                label: 'A',
5945                agent: "alpha".to_owned(),
5946                branch: "magi/x/A".to_owned(),
5947                worktree: wt_a.clone(),
5948                summary: String::new(),
5949                stat: String::new(),
5950                files: 0,
5951                commits: 0,
5952                empty: false,
5953                failed: None,
5954                duration_ms: 0,
5955                folded: false,
5956            },
5957            Candidate {
5958                index: 1,
5959                label: 'B',
5960                agent: "beta".to_owned(),
5961                branch: "magi/x/B".to_owned(),
5962                worktree: wt_b.clone(),
5963                summary: String::new(),
5964                stat: String::new(),
5965                files: 0,
5966                commits: 0,
5967                empty: false,
5968                failed: None,
5969                duration_ms: 0,
5970                folded: false,
5971            },
5972        ];
5973        state.tally = Some(Tally {
5974            first_choice: BTreeMap::from([('A', 1)]),
5975            borda: BTreeMap::new(),
5976            winner: 'A',
5977            rankings: 1,
5978            unanimous_initial: true,
5979            deliberated: false,
5980            changed_votes: 0,
5981            unanimous_final: true,
5982            tie_break: None,
5983            judges: 1,
5984            present: 1,
5985            quorum: 1,
5986            met_quorum: true,
5987            uncontested: None,
5988        });
5989        state.status = RunStatus::Ready;
5990
5991        fold_run(&mut state, false).await.expect("fold_run");
5992
5993        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
5994        assert!(
5995            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
5996            "the unmerged winner's branch survives"
5997        );
5998        assert!(
5999            !state.candidates[0].folded,
6000            "the winner is not marked folded"
6001        );
6002
6003        assert!(!wt_b.exists(), "the loser's worktree is removed");
6004        assert!(
6005            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
6006            "the loser's branch is removed"
6007        );
6008        assert!(state.candidates[1].folded, "the loser is marked folded");
6009    }
6010
6011    /// `status == Ready` used to be read as "this is the harmless
6012    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
6013    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
6014    /// run's PR was closed without merging — and reentering `merge` with
6015    /// `mode` still `Pr` does not know the difference, so it pushed and
6016    /// opened a second pull request. `mode == Local` reproduces the same
6017    /// blind spot without a network call: reentry must not attempt another
6018    /// git merge once this node has already recorded an outcome.
6019    #[tokio::test]
6020    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
6021        let tmp = tempfile::tempdir().expect("tempdir");
6022        let repo = tmp.path().join("repo");
6023        std::fs::create_dir_all(&repo).unwrap();
6024        init_repo(&repo);
6025
6026        let mut config = Config::default();
6027        config.merge.mode = MergeMode::Local;
6028
6029        let mut state = RunState::new(
6030            repo.clone(),
6031            "main".to_owned(),
6032            "deadbeef".to_owned(),
6033            "task".to_owned(),
6034            config,
6035        );
6036        state.candidates = vec![Candidate {
6037            index: 0,
6038            label: 'A',
6039            agent: "alpha".to_owned(),
6040            branch: "does-not-exist".to_owned(),
6041            worktree: repo.clone(),
6042            summary: String::new(),
6043            stat: String::new(),
6044            files: 0,
6045            commits: 0,
6046            empty: false,
6047            failed: None,
6048            duration_ms: 0,
6049            folded: false,
6050        }];
6051        state.tally = Some(Tally {
6052            first_choice: BTreeMap::from([('A', 1)]),
6053            borda: BTreeMap::new(),
6054            winner: 'A',
6055            rankings: 1,
6056            unanimous_initial: true,
6057            deliberated: false,
6058            changed_votes: 0,
6059            unanimous_final: true,
6060            tie_break: None,
6061            judges: 0,
6062            present: 0,
6063            quorum: 0,
6064            met_quorum: true,
6065            uncontested: Some("only candidate A produced a change".to_owned()),
6066        });
6067        state.reviews = vec![ReviewRound {
6068            round: 1,
6069            head: "deadbeef".to_owned(),
6070            verified_head: None,
6071            reviews: Vec::new(),
6072            e2e: Vec::new(),
6073            fix: None,
6074            blocking: 0,
6075            answered: 0,
6076            expected: 0,
6077            clean: true,
6078            verify_retried: false,
6079            e2e_deferred: false,
6080            e2e_defer_reason: None,
6081            progressed: false,
6082            vote_split: false,
6083            reconsideration: Vec::new(),
6084            verdict: None,
6085        }];
6086        state.gate = vec![CommandOutcome {
6087            command: "test".to_owned(),
6088            code: Some(0),
6089            output_tail: String::new(),
6090            duration_ms: 0,
6091            resource_blocked: false,
6092        }];
6093        state.gate_ran = true;
6094        // Reached its conclusion already — e.g. `land` closing the PR without
6095        // merging it, which (like the honest `MergeMode::None` path) leaves
6096        // `status` at `Ready`. The recorded outcome is what actually marks
6097        // this node done.
6098        state.status = RunStatus::Ready;
6099        state.merge = Some(MergeOutcome {
6100            mode: MergeMode::Local,
6101            ok: false,
6102            detail: "already concluded".to_owned(),
6103        });
6104
6105        let mut runner = Runner {
6106            state,
6107            roles: ResolvedRoles {
6108                implementers: Vec::new(),
6109                judges: Vec::new(),
6110                reviewers: Vec::new(),
6111                fixer: None,
6112                conductor: conductor(),
6113            },
6114            sem: Arc::new(Semaphore::new(1)),
6115            pause: Pause::new(),
6116            interrupt: Pause::new(),
6117        };
6118
6119        runner.merge().await.expect("merge");
6120
6121        assert_eq!(
6122            runner.state.status,
6123            RunStatus::Ready,
6124            "a concluded run's status must not change on reentry"
6125        );
6126        assert_eq!(
6127            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
6128            Some("already concluded"),
6129            "merge must not run again once the node already recorded an outcome"
6130        );
6131    }
6132
6133    /// `gate` leaves `state.gate_ran` false both before it has ever run and
6134    /// when its last attempt was resource-blocked (the shared build cache
6135    /// could not be acquired or confirmed fresh in time - see
6136    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
6137    /// `Vec` this also leaves behind used to read as "nothing failed" and let
6138    /// a run merge a tree the gate never actually checked - exactly the case
6139    /// a contended cache produces on every retry until it clears. `merge`
6140    /// must refuse until `gate` has actually recorded an attempt.
6141    #[tokio::test]
6142    async fn merge_refuses_a_gate_that_has_not_actually_run() {
6143        let tmp = tempfile::tempdir().expect("tempdir");
6144        let repo = tmp.path().join("repo");
6145        std::fs::create_dir_all(&repo).unwrap();
6146        init_repo(&repo);
6147
6148        let mut config = Config::default();
6149        config.merge.mode = MergeMode::Local;
6150
6151        let mut state = RunState::new(
6152            repo.clone(),
6153            "main".to_owned(),
6154            "deadbeef".to_owned(),
6155            "task".to_owned(),
6156            config,
6157        );
6158        state.candidates = vec![Candidate {
6159            index: 0,
6160            label: 'A',
6161            agent: "alpha".to_owned(),
6162            branch: "does-not-exist".to_owned(),
6163            worktree: repo.clone(),
6164            summary: String::new(),
6165            stat: String::new(),
6166            files: 0,
6167            commits: 0,
6168            empty: false,
6169            failed: None,
6170            duration_ms: 0,
6171            folded: false,
6172        }];
6173        state.tally = Some(Tally {
6174            first_choice: BTreeMap::from([('A', 1)]),
6175            borda: BTreeMap::new(),
6176            winner: 'A',
6177            rankings: 1,
6178            unanimous_initial: true,
6179            deliberated: false,
6180            changed_votes: 0,
6181            unanimous_final: true,
6182            tie_break: None,
6183            judges: 0,
6184            present: 0,
6185            quorum: 0,
6186            met_quorum: true,
6187            uncontested: Some("only candidate A produced a change".to_owned()),
6188        });
6189        state.reviews = vec![ReviewRound {
6190            round: 1,
6191            head: "deadbeef".to_owned(),
6192            verified_head: None,
6193            reviews: Vec::new(),
6194            e2e: Vec::new(),
6195            fix: None,
6196            blocking: 0,
6197            answered: 0,
6198            expected: 0,
6199            clean: true,
6200            verify_retried: false,
6201            e2e_deferred: false,
6202            e2e_defer_reason: None,
6203            progressed: false,
6204            vote_split: false,
6205            reconsideration: Vec::new(),
6206            verdict: None,
6207        }];
6208        // The point: `gate` has not recorded anything yet.
6209        state.gate = Vec::new();
6210        state.gate_ran = false;
6211        state.status = RunStatus::Gating;
6212
6213        let mut runner = Runner {
6214            state,
6215            roles: ResolvedRoles {
6216                implementers: Vec::new(),
6217                judges: Vec::new(),
6218                reviewers: Vec::new(),
6219                fixer: None,
6220                conductor: conductor(),
6221            },
6222            sem: Arc::new(Semaphore::new(1)),
6223            pause: Pause::new(),
6224            interrupt: Pause::new(),
6225        };
6226
6227        runner.merge().await.expect("merge");
6228
6229        assert!(
6230            runner.state.merge.is_none(),
6231            "an empty gate must never be read as a passing one: {:?}",
6232            runner.state.merge
6233        );
6234    }
6235
6236    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
6237    /// commands configured and `merge.mode` is `none` (a review-only run).
6238    /// `gate` must still record a real attempt — zero commands, vacuously
6239    /// passed — rather than leaving `state.gate` empty in a way `merge`
6240    /// cannot tell apart from "never ran"; otherwise the run reaches
6241    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
6242    #[tokio::test]
6243    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
6244        let tmp = tempfile::tempdir().expect("tempdir");
6245        let repo = tmp.path().join("repo");
6246        std::fs::create_dir_all(&repo).unwrap();
6247        init_repo(&repo);
6248
6249        // Default config: `verify.gate` empty, `merge.mode` is `none`.
6250        let config = Config::default();
6251
6252        let mut state = RunState::new(
6253            repo.clone(),
6254            "main".to_owned(),
6255            "deadbeef".to_owned(),
6256            "task".to_owned(),
6257            config,
6258        );
6259        state.candidates = vec![Candidate {
6260            index: 0,
6261            label: 'A',
6262            agent: "alpha".to_owned(),
6263            branch: "does-not-exist".to_owned(),
6264            worktree: repo.clone(),
6265            summary: String::new(),
6266            stat: String::new(),
6267            files: 0,
6268            commits: 0,
6269            empty: false,
6270            failed: None,
6271            duration_ms: 0,
6272            folded: false,
6273        }];
6274        state.tally = Some(Tally {
6275            first_choice: BTreeMap::from([('A', 1)]),
6276            borda: BTreeMap::new(),
6277            winner: 'A',
6278            rankings: 1,
6279            unanimous_initial: true,
6280            deliberated: false,
6281            changed_votes: 0,
6282            unanimous_final: true,
6283            tie_break: None,
6284            judges: 0,
6285            present: 0,
6286            quorum: 0,
6287            met_quorum: true,
6288            uncontested: Some("only candidate A produced a change".to_owned()),
6289        });
6290        state.reviews = vec![ReviewRound {
6291            round: 1,
6292            head: "deadbeef".to_owned(),
6293            verified_head: None,
6294            reviews: Vec::new(),
6295            e2e: Vec::new(),
6296            fix: None,
6297            blocking: 0,
6298            answered: 0,
6299            expected: 0,
6300            clean: true,
6301            verify_retried: false,
6302            e2e_deferred: false,
6303            e2e_defer_reason: None,
6304            progressed: false,
6305            vote_split: false,
6306            reconsideration: Vec::new(),
6307            verdict: None,
6308        }];
6309
6310        let mut runner = Runner {
6311            state,
6312            roles: ResolvedRoles {
6313                implementers: Vec::new(),
6314                judges: Vec::new(),
6315                reviewers: Vec::new(),
6316                fixer: None,
6317                conductor: conductor(),
6318            },
6319            sem: Arc::new(Semaphore::new(1)),
6320            pause: Pause::new(),
6321            interrupt: Pause::new(),
6322        };
6323
6324        runner.gate().await.expect("gate");
6325        assert!(
6326            runner.state.gate_ran,
6327            "zero configured commands is still a real attempt, not an unrun gate"
6328        );
6329        assert!(runner.state.gate.is_empty());
6330        assert_eq!(runner.state.gate_status(), GateStatus::PassedWithNoCommands);
6331        assert_ne!(
6332            runner.state.status,
6333            RunStatus::Blocked,
6334            "a gate with nothing to check must not read as failed"
6335        );
6336
6337        runner.merge().await.expect("merge");
6338        assert_eq!(
6339            runner.state.status,
6340            RunStatus::Ready,
6341            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
6342        );
6343    }
6344
6345    /// `Config::cache_dir` is derived from `verify.e2e` as well as
6346    /// `verify.gate` (so the e2e leg and the final gate never build against
6347    /// different directories). With zero `verify.gate` commands but a
6348    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
6349    /// that lease before discovering it had nothing to run - so a repo with
6350    /// no gate commands could come back `resource_blocked` (and therefore
6351    /// still `gate_ran == false`) on nothing but an unrelated run holding the
6352    /// cache, exactly the contention this run's own zero commands could
6353    /// never have touched. `gate` must recognise there is nothing to check
6354    /// before it ever asks for the lease.
6355    #[tokio::test]
6356    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
6357        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
6358        let home = crate::run::home();
6359
6360        let tmp = tempfile::tempdir().expect("tempdir");
6361        let repo = tmp.path().join("repo");
6362        std::fs::create_dir_all(&repo).unwrap();
6363        init_repo(&repo);
6364        // Unique to this test, so holding its lease cannot collide with
6365        // another test sharing the same process-wide `home`.
6366        let cache_dir = tmp.path().join("target");
6367
6368        let mut config = Config::default();
6369        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
6370        // `verify.gate` stays empty (the default). Bounded so a regression
6371        // that does start waiting fails the test in seconds, not hangs it.
6372        config.graph.timeout_verify = Some(2);
6373
6374        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
6375        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
6376            .expect("no io error acquiring directly")
6377        {
6378            crate::cache::AcquireOutcome::Acquired(g) => g,
6379            crate::cache::AcquireOutcome::Busy(b) => {
6380                panic!("expected the direct acquire to win the lease first: {b:?}")
6381            }
6382        };
6383
6384        let mut state = RunState::new(
6385            repo.clone(),
6386            "main".to_owned(),
6387            "deadbeef".to_owned(),
6388            "task".to_owned(),
6389            config,
6390        );
6391        state.candidates = vec![Candidate {
6392            index: 0,
6393            label: 'A',
6394            agent: "alpha".to_owned(),
6395            branch: "does-not-exist".to_owned(),
6396            worktree: repo.clone(),
6397            summary: String::new(),
6398            stat: String::new(),
6399            files: 0,
6400            commits: 0,
6401            empty: false,
6402            failed: None,
6403            duration_ms: 0,
6404            folded: false,
6405        }];
6406        state.tally = Some(Tally {
6407            first_choice: BTreeMap::from([('A', 1)]),
6408            borda: BTreeMap::new(),
6409            winner: 'A',
6410            rankings: 1,
6411            unanimous_initial: true,
6412            deliberated: false,
6413            changed_votes: 0,
6414            unanimous_final: true,
6415            tie_break: None,
6416            judges: 0,
6417            present: 0,
6418            quorum: 0,
6419            met_quorum: true,
6420            uncontested: Some("only candidate A produced a change".to_owned()),
6421        });
6422        state.reviews = vec![ReviewRound {
6423            round: 1,
6424            head: "deadbeef".to_owned(),
6425            verified_head: None,
6426            reviews: Vec::new(),
6427            e2e: Vec::new(),
6428            fix: None,
6429            blocking: 0,
6430            answered: 0,
6431            expected: 0,
6432            clean: true,
6433            verify_retried: false,
6434            e2e_deferred: false,
6435            e2e_defer_reason: None,
6436            progressed: false,
6437            vote_split: false,
6438            reconsideration: Vec::new(),
6439            verdict: None,
6440        }];
6441
6442        let mut runner = Runner {
6443            state,
6444            roles: ResolvedRoles {
6445                implementers: Vec::new(),
6446                judges: Vec::new(),
6447                reviewers: Vec::new(),
6448                fixer: None,
6449                conductor: conductor(),
6450            },
6451            sem: Arc::new(Semaphore::new(1)),
6452            pause: Pause::new(),
6453            interrupt: Pause::new(),
6454        };
6455
6456        let started = std::time::Instant::now();
6457        runner.gate().await.expect("gate");
6458        assert!(
6459            started.elapsed() < Duration::from_secs(1),
6460            "a gate with nothing to run must never wait on a lease it never needed"
6461        );
6462        assert!(
6463            runner.state.gate_ran,
6464            "zero commands is still a real, immediate attempt"
6465        );
6466        assert!(runner.state.gate.is_empty());
6467        assert_ne!(
6468            runner.state.status,
6469            RunStatus::Blocked,
6470            "must not read as resource-blocked on a lease it never asked for"
6471        );
6472    }
6473
6474    #[tokio::test]
6475    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
6476        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
6477        let tmp = tempfile::tempdir().expect("tempdir");
6478        let repo = tmp.path().join("repo");
6479        std::fs::create_dir_all(&repo).unwrap();
6480        init_repo(&repo);
6481
6482        let mut config = Config::default();
6483        config.merge.mode = MergeMode::Pr;
6484        config.graph.land = true;
6485        config.graph.land_approval = false;
6486
6487        let mut state = RunState::new(
6488            repo.clone(),
6489            "main".to_owned(),
6490            "deadbeef".to_owned(),
6491            "task".to_owned(),
6492            config,
6493        );
6494        state.candidates = vec![Candidate {
6495            index: 0,
6496            label: 'A',
6497            agent: "alpha".to_owned(),
6498            branch: "does-not-exist".to_owned(),
6499            worktree: repo.clone(),
6500            summary: String::new(),
6501            stat: String::new(),
6502            files: 0,
6503            commits: 0,
6504            empty: false,
6505            failed: None,
6506            duration_ms: 0,
6507            folded: false,
6508        }];
6509        state.tally = Some(Tally {
6510            first_choice: BTreeMap::from([('A', 1)]),
6511            borda: BTreeMap::new(),
6512            winner: 'A',
6513            rankings: 1,
6514            unanimous_initial: true,
6515            deliberated: false,
6516            changed_votes: 0,
6517            unanimous_final: true,
6518            tie_break: None,
6519            judges: 0,
6520            present: 0,
6521            quorum: 0,
6522            met_quorum: true,
6523            uncontested: Some("only candidate A produced a change".to_owned()),
6524        });
6525        state.reviews = vec![ReviewRound {
6526            round: 1,
6527            head: "deadbeef".to_owned(),
6528            verified_head: None,
6529            reviews: Vec::new(),
6530            e2e: Vec::new(),
6531            fix: None,
6532            blocking: 0,
6533            answered: 0,
6534            expected: 0,
6535            clean: true,
6536            verify_retried: false,
6537            e2e_deferred: false,
6538            e2e_defer_reason: None,
6539            progressed: false,
6540            vote_split: false,
6541            reconsideration: Vec::new(),
6542            verdict: None,
6543        }];
6544        state.gate = vec![CommandOutcome {
6545            command: "test".to_owned(),
6546            code: Some(0),
6547            output_tail: String::new(),
6548            duration_ms: 0,
6549            resource_blocked: false,
6550        }];
6551        state.gate_ran = true;
6552        // A first pass through `merge` already pushed and opened this pull
6553        // request; `status` is `Landing` because a previous call into `land`
6554        // parked or was interrupted before it reached a terminal outcome.
6555        state.status = RunStatus::Landing;
6556        state.merge = Some(MergeOutcome {
6557            mode: MergeMode::Pr,
6558            ok: true,
6559            detail: "https://example.invalid/x/y/pull/1".to_owned(),
6560        });
6561
6562        // The Landing-resume shortcut calls `run_land` directly rather than
6563        // through `merge`, which is exactly the call site that used to skip
6564        // `settle_questions` - see the fixture below.
6565        ask_test_home();
6566        let store = ask::Questions::open();
6567        let q = ask_open_question(&store, &state.id);
6568
6569        let mut runner = Runner {
6570            state,
6571            roles: ResolvedRoles {
6572                implementers: Vec::new(),
6573                judges: Vec::new(),
6574                reviewers: Vec::new(),
6575                fixer: None,
6576                conductor: conductor(),
6577            },
6578            sem: Arc::new(Semaphore::new(1)),
6579            pause: Pause::new(),
6580            interrupt: Pause::new(),
6581        };
6582
6583        // `execute`, not `merge` directly: the Landing-resume shortcut lives
6584        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
6585        // exactly because `review_loop` would otherwise clobber the marker
6586        // first.
6587        runner.execute().await.expect("execute");
6588
6589        assert_eq!(
6590            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
6591            Some("https://example.invalid/x/y/pull/1"),
6592            "reentry must not push again or open a second pull request over the \
6593             one `land` is already watching"
6594        );
6595        assert_ne!(
6596            runner.state.status,
6597            RunStatus::Landing,
6598            "land could not actually reach the fake pull request, so it must \
6599             have given up rather than left the run silently parked forever"
6600        );
6601        // `land` could not reach the fake pull request, so it gave up into
6602        // `Blocked` - still resumable, so the question must not have been
6603        // swept just because this branch now also calls `settle_questions`.
6604        assert_eq!(runner.state.status, RunStatus::Blocked);
6605        assert!(
6606            store.get(&q.id).unwrap().status.open(),
6607            "Blocked is still alive; settle_questions must have been a no-op here"
6608        );
6609    }
6610
6611    fn state_with_round(round: ReviewRound) -> RunState {
6612        let mut s = RunState::new(
6613            PathBuf::from("/repo"),
6614            "main".to_owned(),
6615            "abc1234".to_owned(),
6616            "add retries".to_owned(),
6617            Config::default(),
6618        );
6619        s.reviews = vec![round];
6620        s
6621    }
6622
6623    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
6624        crate::verdict::Finding {
6625            id: id.to_owned(),
6626            severity,
6627            file: None,
6628            line: None,
6629            title: title.to_owned(),
6630            detail: String::new(),
6631        }
6632    }
6633
6634    #[test]
6635    fn pr_body_names_open_findings_and_declined_ones() {
6636        let round = ReviewRound {
6637            round: 2,
6638            head: "deadbee".to_owned(),
6639            verified_head: None,
6640            reviews: vec![ReviewRecord {
6641                reviewer: 1,
6642                agent: "alpha".to_owned(),
6643                summary: String::new(),
6644                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
6645                vote: None,
6646                failed: None,
6647                duration_ms: 0,
6648            }],
6649            e2e: vec![CommandOutcome {
6650                command: "cargo test".to_owned(),
6651                code: Some(0),
6652                output_tail: String::new(),
6653                duration_ms: 0,
6654                resource_blocked: false,
6655            }],
6656            verify_retried: false,
6657            e2e_deferred: false,
6658            e2e_defer_reason: None,
6659            fix: Some(FixRecord {
6660                agent: "alpha".to_owned(),
6661                addressed: Vec::new(),
6662                rejected: vec![crate::verdict::Rejection {
6663                    id: "R1-1-1".to_owned(),
6664                    why: "not reachable from any caller".to_owned(),
6665                }],
6666                notes: String::new(),
6667                committed: true,
6668                failed: None,
6669                duration_ms: 0,
6670                continuation: None,
6671            }),
6672            blocking: 0,
6673            answered: 1,
6674            expected: 1,
6675            clean: false,
6676            progressed: true,
6677            vote_split: false,
6678            reconsideration: Vec::new(),
6679            verdict: None,
6680        };
6681        let state = state_with_round(round);
6682        let body = pr_body(&state, 'A');
6683
6684        assert!(body.contains("add retries"), "the task must still be there");
6685        assert!(body.contains("R2-1-1"), "{body}");
6686        assert!(body.contains("unused import"), "{body}");
6687        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
6688        assert!(
6689            body.contains("not reachable from any caller"),
6690            "the reason it was declined: {body}"
6691        );
6692    }
6693
6694    #[test]
6695    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
6696        let round = ReviewRound {
6697            round: 1,
6698            head: "deadbee".to_owned(),
6699            verified_head: None,
6700            reviews: vec![ReviewRecord {
6701                reviewer: 1,
6702                agent: "alpha".to_owned(),
6703                summary: String::new(),
6704                findings: Vec::new(),
6705                vote: None,
6706                failed: None,
6707                duration_ms: 0,
6708            }],
6709            e2e: Vec::new(),
6710            verify_retried: false,
6711            e2e_deferred: false,
6712            e2e_defer_reason: None,
6713            fix: None,
6714            blocking: 0,
6715            answered: 1,
6716            expected: 1,
6717            clean: true,
6718            progressed: false,
6719            vote_split: false,
6720            reconsideration: Vec::new(),
6721            verdict: None,
6722        };
6723        let state = state_with_round(round);
6724        let body = pr_body(&state, 'A');
6725        assert!(!body.contains("Open review findings"), "{body}");
6726        assert!(!body.contains("Declined"), "{body}");
6727    }
6728
6729    #[test]
6730    fn pr_body_titles_itself_from_the_task_not_run_or_candidate() {
6731        let state = RunState::new(
6732            PathBuf::from("/repo"),
6733            "main".to_owned(),
6734            "abc1234".to_owned(),
6735            "add retries".to_owned(),
6736            Config::default(),
6737        );
6738        let body = pr_body(&state, 'A');
6739        let title = body.lines().next().unwrap();
6740
6741        assert_eq!(
6742            title, "add retries",
6743            "the title must be the task, not run/candidate bookkeeping: {body}"
6744        );
6745        assert!(
6746            body.contains(&format!("magi:run/{}", state.id)),
6747            "the run id must still be recoverable from the footer: {body}"
6748        );
6749        assert!(
6750            body.contains("magi:candidate-a"),
6751            "the candidate must still be recoverable from the footer: {body}"
6752        );
6753    }
6754
6755    #[test]
6756    fn pr_body_never_titles_itself_off_a_blank_first_line() {
6757        let leading_blank = RunState::new(
6758            PathBuf::from("/repo"),
6759            "main".to_owned(),
6760            "abc1234".to_owned(),
6761            "\n\n  \nadd retries\n\ndetails".to_owned(),
6762            Config::default(),
6763        );
6764        let body = pr_body(&leading_blank, 'A');
6765        assert_eq!(
6766            body.lines().next(),
6767            Some("add retries"),
6768            "a leading blank line must not become an empty title: {body}"
6769        );
6770
6771        let whitespace_only = RunState::new(
6772            PathBuf::from("/repo"),
6773            "main".to_owned(),
6774            "abc1234".to_owned(),
6775            "   \n  \n".to_owned(),
6776            Config::default(),
6777        );
6778        let body = pr_body(&whitespace_only, 'A');
6779        let title = body.lines().next().unwrap_or_default();
6780        assert!(
6781            !title.is_empty(),
6782            "a whitespace-only instruction must still fall back to a non-empty title: {body}"
6783        );
6784    }
6785
6786    #[test]
6787    fn manual_merge_command_matches_the_configured_style() {
6788        let repo = Path::new("/repo");
6789        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
6790
6791        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
6792        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
6793
6794        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
6795        assert_eq!(
6796            squash,
6797            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
6798             \"Merge magi run 0832 (candidate A)\""
6799        );
6800
6801        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
6802        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
6803    }
6804
6805    #[test]
6806    fn a_nudge_gets_a_quarter_of_the_budget() {
6807        // The judge and implement budgets magi ships with.
6808        assert_eq!(retry_budget(secs(1200), true), secs(300));
6809        assert_eq!(retry_budget(secs(3600), true), secs(900));
6810    }
6811
6812    #[test]
6813    fn a_resent_prompt_keeps_the_whole_budget() {
6814        // The seat kept no context, so the retry is the original job again and
6815        // shortening it would only guarantee a second failure.
6816        assert_eq!(retry_budget(secs(1200), false), secs(1200));
6817        assert_eq!(retry_budget(secs(60), false), secs(60));
6818    }
6819
6820    #[test]
6821    fn the_floor_never_exceeds_the_original_budget() {
6822        // A short configured timeout must not be *raised* by the floor: the
6823        // operator asked for a bound, and a retry may not outlast the attempt
6824        // it is retrying.
6825        assert_eq!(retry_budget(secs(60), true), secs(60));
6826        assert_eq!(retry_budget(secs(480), true), secs(120));
6827        assert_eq!(retry_budget(secs(0), true), secs(0));
6828    }
6829
6830    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
6831        agent::CommandEvidence {
6832            id: "item1".to_owned(),
6833            description: "cargo test".to_owned(),
6834            exit_code,
6835            result_summary: String::new(),
6836            source: "codex".to_owned(),
6837        }
6838    }
6839
6840    #[test]
6841    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
6842        // No evidence is not the same fact as unconfirmed evidence: a
6843        // backend with no adapter, or a reply that ran no commands at all,
6844        // must not be misread as carrying a dangling job.
6845        assert!(!has_unconfirmed_command(&[]));
6846    }
6847
6848    #[test]
6849    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
6850        // Deliberately not a check on the exit code's *value*: a fixer
6851        // legitimately runs something that fails mid-iteration before it
6852        // succeeds, and that must never by itself reopen a valid report.
6853        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
6854        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
6855        assert!(!has_unconfirmed_command(&[
6856            evidence(Some(0)),
6857            evidence(Some(101))
6858        ]));
6859    }
6860
6861    #[test]
6862    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
6863        assert!(has_unconfirmed_command(&[
6864            evidence(Some(0)),
6865            evidence(None)
6866        ]));
6867    }
6868}