Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::git;
39use crate::land;
40use crate::proc::Quiet as _;
41use crate::prompt::{
42    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
43};
44use crate::run::{
45    BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
46    MergeOutcome, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus,
47    Tally, VoteRecord, tail, write_artifact,
48};
49use crate::verdict::{
50    self, FinalVote, FixReport, Position, Proposal, Ranking, Review, ReviewRevote, ReviewVote,
51    Severity,
52};
53
54/// How much verification output is kept and fed back to the fixer.
55const OUTPUT_TAIL: usize = 8_000;
56
57/// Bytes of a failing command's output kept in an event, so the reason a run
58/// stopped is readable from the report without opening `run.json`.
59const EVENT_OUTPUT_TAIL: usize = 2_000;
60
61/// Consecutive review rounds with no tree progress (see
62/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
63/// instead of spending the rest of the round budget.
64///
65/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
66/// legitimately finds nothing left to change (its previous round's fix already
67/// covered it, and this round's reviewers re-raised only nits) looks the same
68/// as one that is spinning, for exactly one round. Two in a row is where the
69/// two stop being distinguishable, and a review round on this workload has
70/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
71/// third attempt at a tree that has not moved twice running is pure cost.
72/// This does not touch `review_rounds` itself, which stays the operator's
73/// call.
74pub(crate) const STAGNANT_LIMIT: usize = 2;
75
76/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
77/// a base that moved before giving up and leaving the run `Blocked` for a
78/// person.
79///
80/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
81/// that keeps moving faster than a run can catch it is not something more
82/// rebasing fixes, it is a person's call. Not the same *number as*
83/// `land_rounds` - this budget is spent before a pull request exists, land's
84/// after - but bounded for the identical reason, so it uses the same
85/// default. Counted across both call sites in [`Runner::finish_after_tally`]
86/// (once before review, once before the gate), because either one finding
87/// the base still moving is the same signal.
88const BASE_SYNC_ROUNDS: usize = 4;
89
90/// One queued agent invocation.
91///
92/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
93/// CLI hung up on its own stream is asked again from the same job rather than
94/// rebuilt from scratch. See [`Runner::resume_undelivered`].
95#[derive(Clone)]
96struct SeatJob {
97    spec: AgentSpec,
98    seat: SeatState,
99    cwd: PathBuf,
100    prompt: String,
101    timeout: Duration,
102    allow_write: bool,
103    sessions: bool,
104    artifacts: PathBuf,
105    stem: String,
106}
107
108/// How the graph reads one agent invocation.
109///
110/// Quota is split out from an ordinary failure on purpose: a rate-limited call
111/// is known to fail again if retried now, so the retry loop must not spend an
112/// attempt on it. `Dropped` is split out for the opposite reason: unlike
113/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
114/// error JSON, never the agent's answer — a caller that matched only
115/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
116/// left to read that JSON as if it were usable output. `resume_undelivered`
117/// is the only caller that acts on it; everywhere else it is reported like an
118/// ordinary failure.
119enum AgentOutcome {
120    /// A usable output.
121    Ok(AgentOutput),
122    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
123    Quota(AgentOutput),
124    /// The CLI hung up on its own stream after billed work. See
125    /// [`agent::AgentOutput::work_undelivered`].
126    Dropped(AgentOutput),
127    /// Any other failure: a timeout, a bad exit code, an empty reply.
128    Failed(String),
129}
130
131/// A request to park the run at its next node boundary.
132///
133/// Cloning is how the request travels: the loop keeps one handle and hands a
134/// clone to each [`Runner`], and every clone points at the same flag. There
135/// is no channel because there is nothing to send - the only message is
136/// "park", it is idempotent, and a flag cannot be missed by a receiver that
137/// was not listening yet.
138///
139/// The boundary is what makes this cheap. Every node writes the run's state
140/// before the next one starts, and every node skips what is already recorded:
141/// `prep` returns early once candidates exist, `implement` asks only the seats
142/// with nothing on disk, `judge` returns early once judgements exist. So a
143/// parked run resumes into exactly the node it stopped before, and no agent
144/// work is thrown away. Killing the process mid-node, by contrast, loses
145/// whatever the seats in flight had not yet written - which for an implement
146/// wave is an hour of paid work.
147#[derive(Debug, Clone, Default)]
148pub struct Pause(Arc<AtomicBool>);
149
150impl Pause {
151    /// A pause nobody has asked for yet.
152    #[must_use]
153    pub fn new() -> Self {
154        Self::default()
155    }
156
157    /// Ask the run to park at its next node boundary. Idempotent.
158    pub fn park(&self) {
159        self.0.store(true, Ordering::SeqCst);
160    }
161
162    /// Has a park been asked for?
163    #[must_use]
164    pub fn parked(&self) -> bool {
165        self.0.load(Ordering::SeqCst)
166    }
167}
168
169/// Drives one run.
170pub struct Runner {
171    /// Run state; public so the CLI can report on it.
172    pub state: RunState,
173    roles: ResolvedRoles,
174    sem: Arc<Semaphore>,
175    /// Set when someone wants the run parked at its next node boundary.
176    pause: Pause,
177}
178
179/// The commit a run branches from: the base branch as the remote has it.
180///
181/// Two failures this replaces. A run used to branch off `HEAD` and so refused
182/// to start on a dirty tree, which made `magi serve` decline every task for as
183/// long as the operator had work in progress - most of the time. Branching off
184/// the *local* base branch fixed that and introduced a worse one: `land` merges
185/// the winner on GitHub, nothing updates the local ref, and the next run
186/// branches off a base missing everything the previous runs landed. Two tasks
187/// in a row from a phone would have had the second silently re-implementing
188/// against stale code and opening a pull request that reverted the first.
189///
190/// Only refs move here - no checkout, no local branch, no merge - so it is safe
191/// with uncommitted work in the tree. A machine with no network still starts:
192/// the fetch may fail and the local tip is used with a warning, because
193/// refusing to run offline is a worse failure than running against a base the
194/// operator can see for themselves.
195///
196/// One function, called by both entry points. Two answers to "where does a run
197/// branch from" is the kind of drift nobody notices until a diff is wrong.
198async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
199    let tracking = format!("{remote}/{base_branch}");
200    let fetched = git::fetch(repo, remote, base_branch).await;
201    if let Ok(out) = &fetched
202        && out.ok()
203        && git::rev_exists(repo, &tracking).await
204    {
205        return git::rev_parse(repo, &tracking).await;
206    }
207    let why = match &fetched {
208        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
209        Ok(_) => format!("{remote} has no {base_branch}"),
210        Err(e) => e.to_string(),
211    };
212    tracing::warn!(
213        "could not read {tracking} ({why}); branching off the local \
214         {base_branch} instead, which may be behind"
215    );
216    git::rev_parse(repo, base_branch).await.with_context(|| {
217        format!(
218            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
219             branch that exists"
220        )
221    })
222}
223
224impl Runner {
225    /// Start a fresh run against `repo`.
226    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
227        let repo = git::toplevel(repo).await?;
228        let missing = agent::missing_programs(&config.agents);
229        if !missing.is_empty() {
230            bail!(
231                "these agent programs are not on PATH: {}. Fix the roster in \
232                 magi.toml or install them.",
233                missing.join(", ")
234            );
235        }
236        let base_branch = match config.merge.base.clone() {
237            Some(b) => b,
238            None => git::current_branch(&repo)
239                .await?
240                .context("HEAD is detached; set [merge] base in magi.toml")?,
241        };
242        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
243        // Still worth saying out loud. The operator's uncommitted work is not
244        // part of this run, and someone watching a candidate fail to use a
245        // change they just made deserves to know why.
246        if !git::is_clean(&repo).await? {
247            tracing::warn!(
248                "{} has uncommitted changes; they are not part of this run, \
249                 which branches off {base_branch} ({})",
250                repo.display(),
251                &base_commit[..base_commit.len().min(8)]
252            );
253        }
254        let roles = config.resolve_roles()?;
255        let max_parallel = config.graph.max_parallel.max(1);
256        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
257        state.event("start", format!("run {} created", state.id));
258        state.save()?;
259        Ok(Self {
260            state,
261            roles,
262            sem: Arc::new(Semaphore::new(max_parallel)),
263            pause: Pause::new(),
264        })
265    }
266
267    /// Open a review-only run against work that already exists on `branch`.
268    ///
269    /// The expensive half of the graph is the implement wave — measured at
270    /// 111 and 134 internal tool-loop turns on this repository, against a
271    /// handful for a judge or a reviewer. The cheap half is worth running on
272    /// hand-written work too, and there was no way to reach it.
273    ///
274    /// No new state and no schema change are needed: a run with **one** viable
275    /// candidate and a tally already decided degrades `execute` to exactly
276    /// review → gate → merge, because `judge` skips a single-candidate field,
277    /// `deliberate` has fewer than two first choices to reconcile, `vote`
278    /// returns early, `tally` is already present and `fold_losers` has no
279    /// losers. Resuming such a run therefore does the right thing as well.
280    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
281        let repo = git::toplevel(repo).await?;
282        let missing = agent::missing_programs(&config.agents);
283        if !missing.is_empty() {
284            bail!(
285                "these agent programs are not on PATH: {}. Fix the roster in \
286                 magi.toml or install them.",
287                missing.join(", ")
288            );
289        }
290        if !git::branch_exists(&repo, branch).await? {
291            bail!("no branch `{branch}` in {}", repo.display());
292        }
293        let base_branch = match config.merge.base.clone() {
294            Some(b) => b,
295            None => git::current_branch(&repo)
296                .await?
297                .context("HEAD is detached; set [merge] base in magi.toml")?,
298        };
299        if base_branch == branch {
300            bail!("`{branch}` is the base branch; there is nothing to review against");
301        }
302        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
303
304        let roles = config.resolve_roles()?;
305        let max_parallel = config.graph.max_parallel.max(1);
306        // The commit subjects are the closest thing to a task statement that
307        // existing work carries, and the reviewers are told as much.
308        let log = git::log_oneline(&repo, &base_commit, branch)
309            .await
310            .unwrap_or_default();
311        let instruction = format!(
312            "Review the work already on branch `{branch}`. There is no task \
313             statement: what the change claims to do is whatever its commits \
314             say.\n\n{}",
315            if log.trim().is_empty() {
316                "(no commit messages)"
317            } else {
318                log.trim()
319            }
320        );
321        let mut state = RunState::new(
322            repo.clone(),
323            base_branch,
324            base_commit.clone(),
325            instruction,
326            config,
327        );
328
329        // An attached worktree, so the fixer's commits land on the branch under
330        // review rather than on a detached head nobody will look at again.
331        let worktree = state.worktree_root().join("under-review");
332        if let Some(parent) = worktree.parent() {
333            tokio::fs::create_dir_all(parent).await.ok();
334        }
335        let path = worktree.to_string_lossy().to_string();
336        git::git(&repo, &["worktree", "add", &path, branch])
337            .await
338            .with_context(|| {
339                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
340            })?;
341
342        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
343            .await
344            .unwrap_or(0);
345        if commits == 0 {
346            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
347        }
348        let files = git::changed_files(&worktree, &base_commit, "HEAD")
349            .await
350            .map(|f| f.len())
351            .unwrap_or(0);
352        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
353            .await
354            .unwrap_or_default();
355
356        state.candidates.push(Candidate {
357            index: 0,
358            label: 'A',
359            // Not an agent id on purpose: nothing in the roster wrote this, and
360            // the stats tables must not credit anyone with a win for it.
361            agent: "(existing branch)".to_owned(),
362            branch: branch.to_owned(),
363            worktree,
364            summary: String::new(),
365            stat,
366            files,
367            commits,
368            empty: false,
369            failed: None,
370            duration_ms: 0,
371            folded: false,
372        });
373        state.tally = Some(Tally {
374            first_choice: BTreeMap::from([('A', 0)]),
375            borda: BTreeMap::new(),
376            winner: 'A',
377            rankings: 0,
378            unanimous_initial: false,
379            deliberated: false,
380            changed_votes: 0,
381            unanimous_final: false,
382            tie_break: None,
383            // No panel sat, so no quorum applies. Zero judges is the correct
384            // number for work that never competed, and must not be reported as
385            // a collapsed panel.
386            judges: 0,
387            present: 0,
388            quorum: 0,
389            met_quorum: true,
390            uncontested: Some("review-only run: nothing competed".to_owned()),
391        });
392        state.status = RunStatus::Reviewing;
393        state.event(
394            "start",
395            format!(
396                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
397                state.id
398            ),
399        );
400        state.save()?;
401        Ok(Self {
402            state,
403            roles,
404            sem: Arc::new(Semaphore::new(max_parallel)),
405            pause: Pause::new(),
406        })
407    }
408
409    /// Reopen an existing run.
410    pub fn resume(id: &str) -> Result<Self> {
411        let state = RunState::load(id)?;
412        let roles = state.config.resolve_roles()?;
413        let max_parallel = state.config.graph.max_parallel.max(1);
414        Ok(Self {
415            state,
416            roles,
417            sem: Arc::new(Semaphore::new(max_parallel)),
418            pause: Pause::new(),
419        })
420    }
421
422    /// Walk the graph to a terminal state, skipping nodes already recorded.
423    pub async fn execute(&mut self) -> Result<()> {
424        // Moving again, so it is no longer parked. Set before the walk rather
425        // than in `resume`, so every way of re-entering the graph clears it
426        // and a card cannot claim a run is waiting to be resumed while the
427        // agents are already working.
428        self.state.parked = false;
429        // Any seat this state still lists as answering belongs to whatever
430        // process last drove this run — this one included, if it crashed
431        // mid-wave. Cleared and flushed immediately, before anything else
432        // runs, so a resume can never show a seat as live when nothing is
433        // asking it anything yet; the node that actually dispatches the next
434        // wave repopulates it.
435        if self.state.clear_active() {
436            self.state.save()?;
437        }
438        // A run that already lost its quorum never resumes into the verdict
439        // machinery: `deliberate` and `vote` would otherwise clobber the
440        // stalled marker back to Voting and the run would keep going past a
441        // verdict that is no longer trustworthy. Everything already recorded is
442        // kept, so the run stays resumable (or foldable) for a human to pick up.
443        //
444        // On --resume the run gets one chance to repair itself: the seats a
445        // rate limit took out are re-asked. If their quota has since reset and
446        // the quorum is restored, the run picks up and finishes; otherwise it
447        // stays stale and still-resumable for a later retry. If it does not
448        // recover, the returned status stays `Stalled` and nothing was
449        // clobbered (the recovery only mutates entries for the lost seats).
450        if self.state.status == RunStatus::Stalled {
451            if self.recover_stall().await? {
452                self.finish_after_tally().await?;
453            } else {
454                // Still below quorum: persist the marker and stay resumable.
455                self.state.save()?;
456            }
457            return Ok(());
458        }
459        // A run parked inside `land` - watching CI, mid fix-round, or
460        // waiting on the owner's merge approval - resumes directly into it,
461        // never back through `prep`. Everything before `merge` already
462        // concluded; that is the only way `status` reaches `Landing` in the
463        // first place. Re-walking `review_loop` first would also be actively
464        // wrong: its own status recomputation (see its doc) treats any
465        // clean round as reason to set `status` to `Gating`, which would
466        // clobber this marker before `merge` ever ran, and this run would
467        // never find its way back into `land` at all.
468        if self.state.status == RunStatus::Landing {
469            self.run_land().await?;
470            // `run_land` may have settled the run right here - CI came back
471            // green and the PR merged, say - without ever passing back
472            // through `merge`'s own trailing call. Whatever it left `status`
473            // as is what this has to read.
474            self.settle_questions();
475            return Ok(());
476        }
477        self.prep().await?;
478        if self.park_here()? {
479            return Ok(());
480        }
481        self.advise().await?;
482        if self.park_here()? {
483            return Ok(());
484        }
485        self.implement().await?;
486        if self.park_here()? {
487            return Ok(());
488        }
489        self.judge().await?;
490        if self.park_here()? {
491            return Ok(());
492        }
493        self.deliberate().await?;
494        if self.park_here()? {
495            return Ok(());
496        }
497        self.vote().await?;
498        if self.park_here()? {
499            return Ok(());
500        }
501        self.tally()?;
502        // A verdict that lost its quorum is not trustworthy: do not review,
503        // gate, or merge on it. Everything already done is kept, so the run
504        // stays resumable (or foldable); the human can replace the agent that
505        // ran out of quota and pick it up.
506        if self.state.status == RunStatus::Stalled {
507            // Persist the stalled marker now — the normal end-of-execute save
508            // below is below this early return, and without it a resumed run
509            // would reload a pre-tally status and keep going.
510            self.state.save()?;
511            return Ok(());
512        }
513        self.finish_after_tally().await?;
514        Ok(())
515    }
516
517    /// Park here if asked to, recording it in the run's own timeline.
518    ///
519    /// Returns whether the caller should stop walking the graph. The state is
520    /// saved either way by the node that just finished; this adds the event so
521    /// the operator's card says why a run that is neither finished nor moving
522    /// is sitting where it is.
523    fn park_here(&mut self) -> Result<bool> {
524        if !self.pause.parked() {
525            return Ok(false);
526        }
527        self.state.event(
528            "park",
529            format!(
530                "parked after `{}` — resume to carry on from here",
531                self.state.status.as_str()
532            ),
533        );
534        self.state.parked = true;
535        self.state.save()?;
536        Ok(true)
537    }
538
539    /// Hand the runner a pause to watch.
540    pub fn on_pause(&mut self, pause: Pause) {
541        self.pause = pause;
542    }
543
544    /// Abandon this run's own open questions, once `status` has actually
545    /// settled rather than merely paused.
546    ///
547    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
548    /// either back up with the candidates, the review round and the seat
549    /// sessions already on disk, so a question an implementer asked mid-round
550    /// may still get a real answer read by a real resume. Only the three
551    /// statuses `resumable` excludes are actually final: the run merged, or
552    /// it reached `Ready` with nothing left to do, or it failed outright with
553    /// no established point to continue from. In every one of those the seat
554    /// that asked is gone for good, exactly like the run being deleted under
555    /// `magi run rm` - so the same cleanup applies, worded for what actually
556    /// happened instead of "the run was deleted".
557    ///
558    /// Best-effort and silent on success: called from every place `status`
559    /// can land on one of those three, including ones a resumed run revisits,
560    /// so it must cost nothing when there was nothing open to begin with.
561    fn settle_questions(&mut self) {
562        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
563            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
564        }
565    }
566
567    /// The tail of the graph after a trustworthy tally: fold losers, review,
568    /// gate, merge, and persist.
569    async fn finish_after_tally(&mut self) -> Result<()> {
570        self.fold_losers().await?;
571        // Before review starts, and again right before the gate: a run's
572        // review rounds can themselves take long enough for the base to move
573        // a second time, and the gate is the one node whose "green" gets
574        // acted on.
575        self.sync_to_base().await?;
576        self.review_loop().await?;
577        self.sync_to_base().await?;
578        self.gate().await?;
579        self.merge().await?;
580        self.state.save()?;
581        Ok(())
582    }
583
584    // ---------------------------------------------------------------- prep
585
586    async fn prep(&mut self) -> Result<()> {
587        if !self.state.candidates.is_empty() {
588            return Ok(());
589        }
590        self.state.status = RunStatus::Prep;
591        let repo = self.state.repo.clone();
592        let base = self.state.base_commit.clone();
593        let root = self.state.worktree_root();
594        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
595
596        // The hook is the write-time half of the blindness contract; the
597        // presentation filter in `blind` is the half that cannot be bypassed.
598        let hooks_dir = self.state.dir().join("hooks");
599        if self.state.config.blind.commit_msg_hook {
600            std::fs::create_dir_all(&hooks_dir)
601                .with_context(|| format!("create {}", hooks_dir.display()))?;
602            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
603            let path = hooks_dir.join("commit-msg");
604            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
605            make_executable(&path)?;
606            // Ref-counted rather than a plain idempotent set: with more than
607            // one run able to be in flight in the same repository at once
608            // (see `Config::daemon.max_concurrent_runs`), a bare "already
609            // true?" check cannot tell "another run of mine still needs
610            // this" from "nobody does", and the run that happens to finish
611            // first would disable the hook out from under a sibling still
612            // relying on it.
613            git::acquire_worktree_config(&repo).await?;
614            self.state.enabled_worktree_config = true;
615        }
616
617        for (index, (spec, label)) in self
618            .roles
619            .implementers
620            .clone()
621            .into_iter()
622            .zip(labels)
623            .enumerate()
624        {
625            let branch = self.state.branch_for(label);
626            let worktree = root.join(format!("cand-{label}"));
627            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
628            if self.state.config.blind.commit_msg_hook {
629                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
630            }
631            git::local_exclude(&worktree, "/.magi/").await?;
632            self.state.candidates.push(Candidate {
633                index,
634                label,
635                agent: spec.id.clone(),
636                branch,
637                worktree,
638                summary: String::new(),
639                stat: String::new(),
640                files: 0,
641                commits: 0,
642                empty: false,
643                failed: None,
644                duration_ms: 0,
645                folded: false,
646            });
647        }
648
649        for j in 1..=self.roles.judges.len() {
650            let wt = root.join(format!("judge-{j}"));
651            if !wt.exists() {
652                git::worktree_add_detached(&repo, &wt, &base).await?;
653            }
654        }
655
656        // Disposable, detached checkouts for the design-deliberation stage's
657        // advisor seats — the same shape as the judges' above, at the same
658        // base commit, since advisors also only ever read. Sized off the
659        // configured count directly rather than a resolved roster: unlike
660        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
661        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
662        // `prep` has no `ResolvedRoles` field to read a count from here.
663        if self.state.config.graph.advise {
664            for k in 1..=self.state.config.graph.advisors {
665                let wt = root.join(format!("advisor-{k}"));
666                if !wt.exists() {
667                    git::worktree_add_detached(&repo, &wt, &base).await?;
668                }
669            }
670        }
671
672        // A judge cannot tell it is looking at its own patch — the seats keep
673        // separate conversations — but a panel that shares agents with the
674        // field is less independent than it looks, and that is worth saying out
675        // loud once per run rather than leaving it in the config.
676        let authors: Vec<&str> = self
677            .roles
678            .implementers
679            .iter()
680            .map(|a| a.id.as_str())
681            .collect();
682        let overlap: Vec<String> = self
683            .roles
684            .judges
685            .iter()
686            .enumerate()
687            .filter(|(_, j)| authors.contains(&j.id.as_str()))
688            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
689            .collect();
690        if !overlap.is_empty() {
691            let note = format!(
692                "{} also authored a candidate; blind, but the panel is less \
693                 independent than {} distinct agents would be",
694                overlap.join(", "),
695                self.roles.judges.len()
696            );
697            self.state.event("prep", note);
698        }
699
700        self.state.event(
701            "prep",
702            format!(
703                "{} candidates, {} judges, base {} ({})",
704                self.state.candidates.len(),
705                self.roles.judges.len(),
706                &self.state.base_commit[..7.min(self.state.base_commit.len())],
707                self.state.base_branch
708            ),
709        );
710        self.state.status = RunStatus::Implementing;
711        self.state.save()?;
712        Ok(())
713    }
714
715    // -------------------------------------------------------------- advise
716
717    /// The design-deliberation stage: independent, read-only advisor seats
718    /// each sketch a design before any implementer touches the repository,
719    /// and (when at least one produced a usable proposal) a synthesis seat
720    /// blends them into a brief `implement` carries in every candidate's
721    /// prompt.
722    ///
723    /// `[graph] advise` is the on/off switch, on by default; `[graph]
724    /// advisors` is the proposal count. Everything here is best-effort and
725    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
726    /// that cannot reach quota, or a synthesis seat that produced nothing
727    /// usable all leave `implement` exactly as it was before this stage
728    /// existed — the task instruction alone — rather than failing the whole
729    /// competition over an enrichment stage. Every outcome is still recorded
730    /// as an event, so a run that got nothing from this stage says why.
731    ///
732    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
733    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
734    /// resumed run whose stage failed would re-run it, and re-spend the
735    /// agent calls, on every reentry before `implement`.
736    ///
737    /// Also skipped once any candidate shows implementation progress — the
738    /// exact predicate `implement` itself uses to decide a candidate is no
739    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
740    /// is not enough: a run created by an older binary that predates this
741    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
742    /// an already-`Implementing`-or-later run under this build would
743    /// otherwise walk straight back through `prep` (a no-op once candidates
744    /// exist) into this node and spawn every advisor seat against worktrees
745    /// `prep` never recreated — after implementation has already started,
746    /// which is exactly the invariant this stage exists to guarantee.
747    async fn advise(&mut self) -> Result<()> {
748        let implement_untouched = self
749            .state
750            .candidates
751            .iter()
752            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
753        if !self.state.config.graph.advise || self.state.advise_attempted {
754            return Ok(());
755        }
756        if !implement_untouched {
757            self.state.event(
758                "advise",
759                "skipping the design-deliberation stage: at least one \
760                 candidate already shows implementation progress, so this \
761                 run is past the point the stage exists to run before"
762                    .to_owned(),
763            );
764            self.state.advise_attempted = true;
765            self.state.save()?;
766            return Ok(());
767        }
768        let run_id = self.state.id.clone();
769        let prompts = self.state.config.prompts.clone();
770        let instruction = self.state.instruction.clone();
771        let language = self.state.config.graph.language.clone();
772        let root = self.state.worktree_root();
773        let n = self.state.config.graph.advisors;
774        let where_recorded = self.state.dir().join("run.json");
775
776        let seats = match self.state.config.advisors() {
777            Ok(seats) if !seats.is_empty() => seats,
778            Ok(_) => {
779                self.state.event(
780                    "advise",
781                    format!(
782                        "[graph] advisors is 0; skipping the design-deliberation \
783                         stage and continuing without a synthesis brief (see {})",
784                        where_recorded.display()
785                    ),
786                );
787                self.state.advise_attempted = true;
788                self.state.save()?;
789                return Ok(());
790            }
791            Err(e) => {
792                self.state.event(
793                    "advise",
794                    format!(
795                        "could not resolve advisor seats ({e:#}); continuing \
796                         without a design-deliberation brief (see {})",
797                        where_recorded.display()
798                    ),
799                );
800                self.state.advise_attempted = true;
801                self.state.save()?;
802                return Ok(());
803            }
804        };
805
806        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
807        let artifacts = agent::artifacts_dir(&self.state.dir());
808        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
809
810        let mut jobs = Vec::new();
811        for (i, spec) in seats.iter().cloned().enumerate() {
812            let seat_key = format!("advisor-{}", i + 1);
813            let seat = self.seat(&seat_key, &spec.id);
814            jobs.push(SeatJob {
815                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
816                spec,
817                seat,
818                cwd: worktrees[i % worktrees.len()].clone(),
819                timeout,
820                allow_write: false,
821                sessions: false,
822                artifacts: artifacts.clone(),
823                stem: seat_key,
824            });
825        }
826
827        self.state.event(
828            "advise",
829            format!(
830                "{} advisor seat(s) sketching a design in parallel",
831                jobs.len()
832            ),
833        );
834        let mut quota_losses = Vec::new();
835        let cache = self.state.config.cache_dir();
836        let ctx = WaveCtx {
837            run: &run_id,
838            node: "advise",
839            prompts: &prompts,
840            cache: cache.as_deref(),
841        };
842        let results = ask_json_wave::<Proposal>(
843            jobs,
844            Arc::clone(&self.sem),
845            self.state.config.graph.retries,
846            &ctx,
847            &mut quota_losses,
848            &mut self.state,
849            &|p: &Proposal| p.validate(),
850        )
851        .await;
852        self.state.quota.extend(quota_losses);
853
854        let mut records = Vec::with_capacity(results.len());
855        for (i, (seat, res)) in results.into_iter().enumerate() {
856            let agent_id = seat.agent.clone();
857            self.state.seats.insert(seat.key.clone(), seat);
858            match res {
859                Ok((proposal, out)) => {
860                    self.state
861                        .event("advise", format!("advisor-{} proposed a design", i + 1));
862                    records.push(advise::AdvisorRecord::proposed(
863                        i + 1,
864                        agent_id,
865                        proposal,
866                        out.duration_ms,
867                    ));
868                }
869                Err(e) => {
870                    self.state.event(
871                        "advise",
872                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
873                    );
874                    records.push(advise::AdvisorRecord::failed(
875                        i + 1,
876                        agent_id,
877                        e.to_string(),
878                    ));
879                }
880            }
881        }
882
883        let mut advice = advise::Advice {
884            records,
885            synthesis: None,
886        };
887        if advice.proposals().is_empty() {
888            self.state.event(
889                "advise",
890                "no advisor produced a usable proposal; continuing without a \
891                 synthesis brief"
892                    .to_owned(),
893            );
894        } else {
895            match self
896                .synthesize_brief(
897                    &advice,
898                    &instruction,
899                    &language,
900                    &worktrees[0],
901                    &artifacts,
902                    &run_id,
903                    &prompts,
904                    cache.as_deref(),
905                )
906                .await
907            {
908                Ok(Some(text)) => {
909                    self.state.event(
910                        "advise",
911                        "synthesized a design brief for the implementer".to_owned(),
912                    );
913                    advice.synthesis = Some(text);
914                }
915                Ok(None) => {
916                    self.state.event(
917                        "advise",
918                        "the synthesis seat produced nothing usable; continuing \
919                         without a design brief"
920                            .to_owned(),
921                    );
922                }
923                Err(e) => {
924                    self.state.event(
925                        "advise",
926                        format!("could not synthesize a design brief: {e:#}"),
927                    );
928                }
929            }
930        }
931        advise::apply_reflection(&mut advice);
932
933        self.state.advice = Some(advice);
934        self.state.advise_attempted = true;
935        self.state.save()?;
936        Ok(())
937    }
938
939    /// The synthesis seat: reads every advisor's proposal and blends them
940    /// into the design brief `advise` stores on [`RunState::advice`]. Split
941    /// out of [`Runner::advise`] only for readability — it is not called
942    /// anywhere else.
943    ///
944    /// Picked the same way [`crate::talk`]'s standing conversation and
945    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`] with no
946    /// explicit id, rather than a dedicated `[roles]` entry — one more role
947    /// to configure for a seat that runs once per run and, unlike the
948    /// advisors it reads, never needs more than one.
949    #[allow(clippy::too_many_arguments)]
950    async fn synthesize_brief(
951        &mut self,
952        advice: &advise::Advice,
953        instruction: &str,
954        language: &str,
955        cwd: &Path,
956        artifacts: &Path,
957        run_id: &str,
958        prompts: &Prompts,
959        cache: Option<&Path>,
960    ) -> Result<Option<String>> {
961        let spec = agent::pick(&self.state.config.agents, None, &agent::installed)?;
962        let mut seat = self.seat("advise-synthesis", &spec.id);
963        let proposals = advice.proposals();
964        let mut prompt = prompt::with_overlay(
965            prompt::synthesize_brief(instruction, &proposals, language),
966            prompts.overlay("advise"),
967        );
968        if cache.is_some() {
969            prompt.push('\n');
970            prompt.push_str(&prompt::build_cache_note("advise"));
971        }
972        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
973        let out = agent::invoke(
974            &spec,
975            &mut seat,
976            &Invocation {
977                cwd,
978                prompt: &prompt,
979                timeout,
980                allow_write: false,
981                sessions: false,
982                artifacts,
983                stem: "advise-synthesis",
984                run: run_id,
985                node: "advise",
986                cache_dir: cache,
987                attachments: &[],
988            },
989        )
990        .await?;
991        self.state.seats.insert(seat.key.clone(), seat);
992        if !out.usable() {
993            return Ok(None);
994        }
995        let text =
996            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
997        Ok((!text.trim().is_empty()).then_some(text))
998    }
999
1000    // ----------------------------------------------------------- implement
1001
1002    async fn implement(&mut self) -> Result<()> {
1003        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1004        // agent files with `magi task add` name the run that paid for it. The
1005        // prompt overlay is cloned alongside it because the waves borrow it
1006        // while `self` is mutably borrowed by the node's own bookkeeping.
1007        let run_id = self.state.id.clone();
1008        let prompts = self.state.config.prompts.clone();
1009        let todo: Vec<usize> = self
1010            .state
1011            .candidates
1012            .iter()
1013            .enumerate()
1014            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
1015            .map(|(i, _)| i)
1016            .collect();
1017        if todo.is_empty() {
1018            return self.after_implement();
1019        }
1020        self.state.status = RunStatus::Implementing;
1021
1022        let language = self.state.config.graph.language.clone();
1023        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
1024        let sessions = self.state.config.graph.sessions;
1025        let artifacts = agent::artifacts_dir(&self.state.dir());
1026        // The design-deliberation stage's blended brief, when `advise` found
1027        // one — carried into every implementer's prompt the same way
1028        // regardless of which candidate it is.
1029        let brief = self
1030            .state
1031            .advice
1032            .as_ref()
1033            .and_then(|a| a.synthesis.as_deref())
1034            .map(str::to_owned);
1035
1036        let mut jobs = Vec::new();
1037        for &i in &todo {
1038            let (index, label, worktree) = {
1039                let c = &self.state.candidates[i];
1040                (c.index, c.label, c.worktree.clone())
1041            };
1042            let spec = self.roles.implementers[index].clone();
1043            let seat_key = format!("impl-{label}");
1044            let seat = self.seat(&seat_key, &spec.id);
1045            let instruction = self.state.instruction.clone();
1046            jobs.push(SeatJob {
1047                spec,
1048                seat,
1049                prompt: prompt::implement(
1050                    &instruction,
1051                    &worktree.to_string_lossy(),
1052                    &language,
1053                    brief.as_deref(),
1054                ),
1055                cwd: worktree,
1056                timeout,
1057                allow_write: true,
1058                sessions,
1059                artifacts: artifacts.clone(),
1060                stem: format!("impl-{label}"),
1061            });
1062        }
1063
1064        self.state.event(
1065            "implement",
1066            format!("{} candidates in parallel", jobs.len()),
1067        );
1068        // Kept so a seat whose CLI hung up can be asked again from the same
1069        // job: `wave` consumes what it is given.
1070        let sent = jobs.clone();
1071        let cache = self.state.config.cache_dir();
1072        let ctx = WaveCtx {
1073            run: &run_id,
1074            node: "implement",
1075            prompts: &prompts,
1076            cache: cache.as_deref(),
1077        };
1078        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1079        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
1080            .await;
1081
1082        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
1083            let seat_key = seat.key.clone();
1084            self.state.seats.insert(seat.key.clone(), seat);
1085            let label = self.state.candidates[i].label;
1086            let worktree = self.state.candidates[i].worktree.clone();
1087            let base = self.state.base_commit.clone();
1088
1089            let (summary, duration, failed) = match out {
1090                AgentOutcome::Ok(o) => {
1091                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
1092                    let failed = (!o.usable()).then(|| {
1093                        if o.timed_out {
1094                            "agent timed out".to_owned()
1095                        } else {
1096                            format!("agent exited with {:?}", o.exit_code)
1097                        }
1098                    });
1099                    (text, o.duration_ms, failed)
1100                }
1101                // Left un-resumed by `resume_undelivered` (a dirty tree
1102                // already rescues the work, or there was no session left to
1103                // resume into) — reported like the ordinary failure it is,
1104                // never as if `o.text` (the CLI's raw error JSON) were an
1105                // answer.
1106                AgentOutcome::Dropped(o) => {
1107                    let why = o
1108                        .dropped
1109                        .as_ref()
1110                        .map(|d| d.why.as_str())
1111                        .unwrap_or("the CLI ended the stream without delivering its answer");
1112                    (
1113                        String::new(),
1114                        o.duration_ms,
1115                        Some(format!("the CLI dropped the stream ({why})")),
1116                    )
1117                }
1118                AgentOutcome::Quota(o) => {
1119                    self.state.quota.push(QuotaLoss {
1120                        seat: seat_key,
1121                        node: "implement".to_owned(),
1122                        at: Timestamp::now(),
1123                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1124                    });
1125                    (
1126                        String::new(),
1127                        o.duration_ms,
1128                        Some("rate limited (quota); produced no change".to_owned()),
1129                    )
1130                }
1131                AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
1132            };
1133
1134            // Rescue anything the agent edited but never committed: an
1135            // uncommitted candidate would silently be an empty one.
1136            let rescued = git::commit_all(
1137                &worktree,
1138                &format!("magi: candidate {label} (uncommitted work)"),
1139            )
1140            .await
1141            .unwrap_or(false);
1142            let commits = git::commits_ahead(&worktree, &base, "HEAD")
1143                .await
1144                .unwrap_or(0);
1145            let patch = git::diff(&worktree, &base, "HEAD")
1146                .await
1147                .unwrap_or_default();
1148            let stat = git::diff_stat(&worktree, &base, "HEAD")
1149                .await
1150                .unwrap_or_default();
1151            let files = git::changed_files(&worktree, &base, "HEAD")
1152                .await
1153                .map(|f| f.len())
1154                .unwrap_or(0);
1155            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
1156
1157            let c = &mut self.state.candidates[i];
1158            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
1159            c.stat = stat;
1160            c.files = files;
1161            c.commits = commits;
1162            c.duration_ms = duration;
1163            c.empty = commits == 0 || patch.trim().is_empty();
1164            // An agent that failed but still produced a committed change stays
1165            // in the running: the patch is what gets judged, not the exit code.
1166            c.failed = match failed {
1167                Some(_) if c.empty => failed,
1168                _ => None,
1169            };
1170            let note = match (&c.failed, c.empty, rescued) {
1171                (Some(e), _, _) => format!("candidate {label}: {e}"),
1172                (None, true, _) => format!("candidate {label}: no change produced"),
1173                (None, false, true) => {
1174                    format!(
1175                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
1176                    )
1177                }
1178                (None, false, false) => {
1179                    format!("candidate {label}: {files} files, {commits} commits")
1180                }
1181            };
1182            self.state.event("implement", note);
1183            self.state.save()?;
1184        }
1185
1186        self.after_implement()
1187    }
1188
1189    /// Ask again, once, for work a CLI did and then failed to hand over.
1190    ///
1191    /// [`agent::dropped_stream`] recognises the one shape observed: an error
1192    /// status with an empty response and a usage report showing output tokens,
1193    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
1194    /// seven minutes and 14,267 output tokens that arrived as an empty
1195    /// candidate, because `agy`'s own subscriber fell behind and hung up.
1196    ///
1197    /// Two conditions, and both matter:
1198    ///
1199    /// - **Only when the tree is untouched.** Often the agent has already
1200    ///   written its files and only the closing message was lost; the rescue
1201    ///   commit below picks that up and there is nothing to ask for. Re-asking
1202    ///   then would pay for a second implementation of work already on disk.
1203    /// - **Once.** A CLI that drops one stream can drop the next, and this
1204    ///   node is the most expensive in the graph.
1205    ///
1206    /// The re-ask is a resume, not a re-run: `has_context` is true because the
1207    /// dropped reply still carried its `conversation_id`, so the seat is asked
1208    /// to finish what it was doing rather than sent the whole task again. It
1209    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
1210    /// node's - for the same reason a re-ranked judge does: restating finished
1211    /// work is not the work.
1212    ///
1213    /// Unlike a quota this is worth retrying at all: a rate limit fails the
1214    /// same way until it resets, while an abandoned conversation is still
1215    /// there to be picked up.
1216    async fn resume_undelivered(
1217        &mut self,
1218        results: &mut [(usize, SeatState, AgentOutcome)],
1219        sent: &[SeatJob],
1220        prompts: &Prompts,
1221        run_id: &str,
1222    ) {
1223        for (wi, seat, out) in results.iter_mut() {
1224            let Some(dropped) = (match &*out {
1225                AgentOutcome::Dropped(o) => o.dropped.clone(),
1226                _ => None,
1227            }) else {
1228                continue;
1229            };
1230            let Some(job) = sent.get(*wi) else { continue };
1231            // Already on disk? Then only the closing message was lost.
1232            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
1233                self.state.event(
1234                    "implement",
1235                    format!(
1236                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
1237                         work is in the tree",
1238                        seat.key, dropped.output_tokens, dropped.why
1239                    ),
1240                );
1241                continue;
1242            }
1243            // The re-ask only makes sense as a resume: `resume_after_drop`
1244            // says nothing about the task, trusting the seat to still hold it.
1245            // Without a session to resume — sessions disabled, or this CLI's
1246            // drop shape happened not to carry a session id — that prompt
1247            // would open a brand-new conversation with no context at all,
1248            // which is worse than leaving this as the ordinary failure it
1249            // already is.
1250            if !has_context(&job.spec, seat, job.sessions) {
1251                self.state.event(
1252                    "implement",
1253                    format!(
1254                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
1255                         is no session left to resume",
1256                        seat.key, dropped.output_tokens, dropped.why
1257                    ),
1258                );
1259                continue;
1260            }
1261            self.state.event(
1262                "implement",
1263                format!(
1264                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
1265                     conversation",
1266                    seat.key, dropped.output_tokens, dropped.why
1267                ),
1268            );
1269            let mut retry = job.clone();
1270            retry.seat = seat.clone();
1271            retry.prompt = prompt::resume_after_drop(&dropped.why);
1272            retry.timeout = retry_budget(job.timeout, true);
1273            retry.stem = format!("{}-resume", job.stem);
1274            let cache = self.state.config.cache_dir();
1275            let ctx = WaveCtx {
1276                run: run_id,
1277                node: "implement",
1278                prompts,
1279                cache: cache.as_deref(),
1280            };
1281            let (resumed_seat, resumed) =
1282                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
1283            *seat = resumed_seat;
1284            *out = resumed;
1285        }
1286    }
1287
1288    fn after_implement(&mut self) -> Result<()> {
1289        // Scan every candidate patch once the set is complete.
1290        if self.state.leaks.is_empty() {
1291            let cfg = self.state.config.blind.clone();
1292            let mut leaks = Vec::new();
1293            for c in &self.state.candidates {
1294                let Some(patch) =
1295                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
1296                else {
1297                    continue;
1298                };
1299                leaks.extend(blind::scan(
1300                    &format!("candidate {} patch", c.label),
1301                    &patch,
1302                    &cfg.vendor_tokens,
1303                ));
1304            }
1305            if !leaks.is_empty() {
1306                let summary = leaks
1307                    .iter()
1308                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
1309                    .collect::<Vec<_>>()
1310                    .join(", ");
1311                match cfg.on_leak {
1312                    LeakPolicy::Fail => {
1313                        self.state.status = RunStatus::Failed;
1314                        self.state
1315                            .event("blind", format!("vendor text in a patch: {summary}"));
1316                        self.state.leaks = leaks;
1317                        self.state.save()?;
1318                        self.settle_questions();
1319                        bail!(
1320                            "blind.on_leak = \"fail\" and vendor text reached a \
1321                             judged patch: {summary}"
1322                        );
1323                    }
1324                    LeakPolicy::Redact => self.state.event(
1325                        "blind",
1326                        format!("redacting vendor text for judging: {summary}"),
1327                    ),
1328                    LeakPolicy::Warn => self.state.event(
1329                        "blind",
1330                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
1331                    ),
1332                }
1333                self.state.leaks = leaks;
1334            }
1335        }
1336
1337        if self.state.viable().is_empty() {
1338            self.state.status = RunStatus::Failed;
1339            self.state.save()?;
1340            self.settle_questions();
1341            bail!("no candidate produced a change; nothing to judge");
1342        }
1343        self.state.status = RunStatus::Judging;
1344        self.state.save()?;
1345        Ok(())
1346    }
1347
1348    // --------------------------------------------------------------- judge
1349
1350    async fn judge(&mut self) -> Result<()> {
1351        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1352        // agent files with `magi task add` name the run that paid for it. The
1353        // prompt overlay is cloned alongside it because the waves borrow it
1354        // while `self` is mutably borrowed by the node's own bookkeeping.
1355        let run_id = self.state.id.clone();
1356        let prompts = self.state.config.prompts.clone();
1357        if !self.state.judgements.is_empty() || self.state.judge_skipped {
1358            return Ok(());
1359        }
1360        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1361        if viable.len() == 1 {
1362            // Recorded so this is a one-time event: `judgements` stays empty
1363            // either way, which without this flag is indistinguishable from
1364            // "not yet judged" on the next reentry — and status is left
1365            // untouched, so a later node's conclusion (e.g. `Blocked` after
1366            // the review budget ran out) survives a resume instead of being
1367            // clobbered back to `Judging` by this node running again.
1368            self.state.judge_skipped = true;
1369            self.state.event(
1370                "judge",
1371                format!(
1372                    "only candidate {} produced a change; judging skipped",
1373                    viable[0].label
1374                ),
1375            );
1376            self.state.save()?;
1377            return Ok(());
1378        }
1379        self.state.status = RunStatus::Judging;
1380
1381        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1382        let language = self.state.config.graph.language.clone();
1383        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1384        let sessions = self.state.config.graph.sessions;
1385        let artifacts = agent::artifacts_dir(&self.state.dir());
1386        let root = self.state.worktree_root();
1387        let base_short = short(&self.state.base_commit);
1388
1389        let mut jobs = Vec::new();
1390        let mut orders = Vec::new();
1391        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1392            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1393            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1394            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
1395            let seat_key = format!("judge-{}", j + 1);
1396            let seat = self.seat(&seat_key, &spec.id);
1397            jobs.push(SeatJob {
1398                prompt: prompt::judge(
1399                    &self.state.instruction,
1400                    &views,
1401                    self.roles.judges.len(),
1402                    &base_short,
1403                    &language,
1404                ),
1405                spec,
1406                seat,
1407                cwd: root.join(format!("judge-{}", j + 1)),
1408                timeout,
1409                allow_write: false,
1410                sessions,
1411                artifacts: artifacts.clone(),
1412                stem: format!("judge-{}", j + 1),
1413            });
1414        }
1415
1416        self.state.event(
1417            "judge",
1418            format!(
1419                "{} judges ranking {} candidates blind",
1420                jobs.len(),
1421                viable.len()
1422            ),
1423        );
1424        let labels_for_check = labels.clone();
1425        let mut quota_losses = Vec::new();
1426        let cache = self.state.config.cache_dir();
1427        let ctx = WaveCtx {
1428            run: &run_id,
1429            node: "judge",
1430            prompts: &prompts,
1431            cache: cache.as_deref(),
1432        };
1433        let results = ask_json_wave::<Ranking>(
1434            jobs,
1435            Arc::clone(&self.sem),
1436            self.state.config.graph.retries,
1437            &ctx,
1438            &mut quota_losses,
1439            &mut self.state,
1440            &move |r: &Ranking| r.validate(&labels_for_check),
1441        )
1442        .await;
1443        self.state.quota.extend(quota_losses);
1444
1445        for (j, (seat, res)) in results.into_iter().enumerate() {
1446            let agent_id = seat.agent.clone();
1447            self.state.seats.insert(seat.key.clone(), seat);
1448            let mut record = Judgement {
1449                judge: j + 1,
1450                seat: format!("judge-{}", j + 1),
1451                agent: agent_id,
1452                ranking: Vec::new(),
1453                reasons: BTreeMap::new(),
1454                confidence: None,
1455                order: orders[j].clone(),
1456                failed: None,
1457                duration_ms: 0,
1458            };
1459            match res {
1460                Ok((ranking, out)) => {
1461                    record.ranking = ranking.normalized();
1462                    record.reasons = ranking.reasons;
1463                    record.confidence = ranking.confidence;
1464                    record.duration_ms = out.duration_ms;
1465                    self.state.event(
1466                        "judge",
1467                        format!(
1468                            "judge {} ranked {}",
1469                            j + 1,
1470                            record.ranking.iter().collect::<String>()
1471                        ),
1472                    );
1473                }
1474                Err(e) => {
1475                    record.failed = Some(e.to_string());
1476                    self.state
1477                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1478                }
1479            }
1480            self.state.judgements.push(record);
1481            self.state.save()?;
1482        }
1483        Ok(())
1484    }
1485
1486    // ---------------------------------------------------------- deliberate
1487
1488    async fn deliberate(&mut self) -> Result<()> {
1489        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1490        // agent files with `magi task add` name the run that paid for it. The
1491        // prompt overlay is cloned alongside it because the waves borrow it
1492        // while `self` is mutably borrowed by the node's own bookkeeping.
1493        let run_id = self.state.id.clone();
1494        let prompts = self.state.config.prompts.clone();
1495        if !self.state.deliberation.is_empty() {
1496            return Ok(());
1497        }
1498        let tops: Vec<char> = self
1499            .state
1500            .judgements
1501            .iter()
1502            .filter_map(|j| j.ranking.first().copied())
1503            .collect();
1504        let rounds = self.state.config.graph.deliberate_rounds;
1505        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1506            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1507                self.state.event(
1508                    "deliberate",
1509                    format!("judges agreed on {} outright; no deliberation", tops[0]),
1510                );
1511            }
1512            self.state.status = RunStatus::Voting;
1513            self.state.save()?;
1514            return Ok(());
1515        }
1516
1517        self.state.status = RunStatus::Deliberating;
1518        self.state.event(
1519            "deliberate",
1520            format!(
1521                "split: first choices were {} — opening {rounds} round(s)",
1522                tops.iter().collect::<String>()
1523            ),
1524        );
1525
1526        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1527        let language = self.state.config.graph.language.clone();
1528        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1529        let sessions = self.state.config.graph.sessions;
1530        let artifacts = agent::artifacts_dir(&self.state.dir());
1531        let root = self.state.worktree_root();
1532        let base_short = short(&self.state.base_commit);
1533
1534        // Judges argue in sequence so that a turn can answer the one before it;
1535        // that is the difference between deliberation and three parallel
1536        // monologues.
1537        for round in 1..=rounds {
1538            let mut turns: Vec<DeliberationTurn> = Vec::new();
1539            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1540                if self.state.judgements[j].failed.is_some() {
1541                    continue;
1542                }
1543                let seat_key = format!("judge-{}", j + 1);
1544                let mut seat = self.seat(&seat_key, &spec.id);
1545                let transcript = self.transcript(&turns, j);
1546                let context = if has_context(&spec, &seat, sessions) {
1547                    None
1548                } else {
1549                    Some(self.candidate_block(&viable, &base_short))
1550                };
1551                let text = prompt::deliberate(
1552                    &self.state.instruction,
1553                    context.as_deref(),
1554                    &transcript,
1555                    round,
1556                    rounds,
1557                    &language,
1558                );
1559                let job = SeatJob {
1560                    spec,
1561                    seat: seat.clone(),
1562                    prompt: text,
1563                    cwd: root.join(format!("judge-{}", j + 1)),
1564                    timeout,
1565                    allow_write: false,
1566                    sessions,
1567                    artifacts: artifacts.clone(),
1568                    stem: format!("delib-{round}-judge-{}", j + 1),
1569                };
1570                let cache = self.state.config.cache_dir();
1571                let ctx = WaveCtx {
1572                    run: &run_id,
1573                    node: "deliberate",
1574                    prompts: &prompts,
1575                    cache: cache.as_deref(),
1576                };
1577                let (updated, out) =
1578                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1579                seat = updated;
1580                let agent_id = seat.agent.clone();
1581                let seat_key = seat.key.clone();
1582                self.state.seats.insert(seat.key.clone(), seat);
1583                let body = match out {
1584                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1585                    // Never read the CLI's raw error JSON as this judge's
1586                    // position — skip the seat instead, the same as any other
1587                    // failed turn.
1588                    AgentOutcome::Dropped(o) => {
1589                        let why =
1590                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1591                                "the CLI ended the stream without delivering its answer",
1592                            );
1593                        self.state.event(
1594                            "deliberate",
1595                            format!(
1596                                "judge {} skipped: the CLI dropped the stream ({why})",
1597                                j + 1
1598                            ),
1599                        );
1600                        continue;
1601                    }
1602                    AgentOutcome::Quota(o) => {
1603                        self.state.quota.push(QuotaLoss {
1604                            seat: seat_key,
1605                            node: "deliberate".to_owned(),
1606                            at: Timestamp::now(),
1607                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1608                        });
1609                        self.state.event(
1610                            "deliberate",
1611                            format!("judge {} skipped: rate limited (quota)", j + 1),
1612                        );
1613                        continue;
1614                    }
1615                    AgentOutcome::Failed(e) => {
1616                        self.state
1617                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1618                        continue;
1619                    }
1620                };
1621                let tentative = verdict::extract_json::<Position>(&body)
1622                    .ok()
1623                    .and_then(|p| p.tentative)
1624                    .and_then(|s| s.trim().chars().next())
1625                    .map(|c| c.to_ascii_uppercase());
1626                self.state.event(
1627                    "deliberate",
1628                    format!(
1629                        "round {round}: judge {} now favours {}",
1630                        j + 1,
1631                        tentative.map_or("—".to_owned(), |c| c.to_string())
1632                    ),
1633                );
1634                turns.push(DeliberationTurn {
1635                    judge: j + 1,
1636                    agent: agent_id,
1637                    body: blind::sanitize_prose(&body, &self.state.config.blind),
1638                    tentative,
1639                });
1640            }
1641            self.state
1642                .deliberation
1643                .push(DeliberationRound { round, turns });
1644            self.state.save()?;
1645        }
1646
1647        self.state.status = RunStatus::Voting;
1648        self.state.save()?;
1649        Ok(())
1650    }
1651
1652    // ---------------------------------------------------------------- vote
1653
1654    async fn vote(&mut self) -> Result<()> {
1655        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1656        // agent files with `magi task add` name the run that paid for it. The
1657        // prompt overlay is cloned alongside it because the waves borrow it
1658        // while `self` is mutably borrowed by the node's own bookkeeping.
1659        let run_id = self.state.id.clone();
1660        let prompts = self.state.config.prompts.clone();
1661        if !self.state.votes.is_empty() {
1662            return Ok(());
1663        }
1664        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1665        if viable.len() == 1 {
1666            return Ok(());
1667        }
1668        self.state.status = RunStatus::Voting;
1669
1670        let language = self.state.config.graph.language.clone();
1671        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1672        let sessions = self.state.config.graph.sessions;
1673        let artifacts = agent::artifacts_dir(&self.state.dir());
1674        let root = self.state.worktree_root();
1675        let base_short = short(&self.state.base_commit);
1676        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1677
1678        let mut jobs = Vec::new();
1679        let mut seats_at = Vec::new();
1680        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1681            if self
1682                .state
1683                .judgements
1684                .get(j)
1685                .is_some_and(|r| r.failed.is_some())
1686            {
1687                continue;
1688            }
1689            let seat_key = format!("judge-{}", j + 1);
1690            let seat = self.seat(&seat_key, &spec.id);
1691            let mut text = prompt::final_vote(&viable, &language);
1692            if !has_context(&spec, &seat, sessions) {
1693                text = format!(
1694                    "{}\n\n# Candidates\n\n{}",
1695                    text,
1696                    self.candidate_block(&candidates, &base_short)
1697                );
1698            }
1699            jobs.push(SeatJob {
1700                spec,
1701                seat,
1702                prompt: text,
1703                cwd: root.join(format!("judge-{}", j + 1)),
1704                timeout,
1705                allow_write: false,
1706                sessions,
1707                artifacts: artifacts.clone(),
1708                stem: format!("vote-judge-{}", j + 1),
1709            });
1710            seats_at.push(j);
1711        }
1712
1713        self.state.event(
1714            "vote",
1715            format!(
1716                "collecting {} final votes one by one, privately",
1717                jobs.len()
1718            ),
1719        );
1720        let allowed = viable.clone();
1721        let mut quota_losses = Vec::new();
1722        let cache = self.state.config.cache_dir();
1723        let ctx = WaveCtx {
1724            run: &run_id,
1725            node: "vote",
1726            prompts: &prompts,
1727            cache: cache.as_deref(),
1728        };
1729        let results = ask_json_wave::<FinalVote>(
1730            jobs,
1731            Arc::clone(&self.sem),
1732            self.state.config.graph.retries,
1733            &ctx,
1734            &mut quota_losses,
1735            &mut self.state,
1736            &move |v: &FinalVote| match v.label() {
1737                Some(c) if allowed.contains(&c) => Ok(()),
1738                other => bail!("vote {other:?} is not one of {allowed:?}"),
1739            },
1740        )
1741        .await;
1742        self.state.quota.extend(quota_losses);
1743
1744        for (&j, (seat, res)) in seats_at.iter().zip(results) {
1745            let agent_id = seat.agent.clone();
1746            self.state.seats.insert(seat.key.clone(), seat);
1747            let initial = self
1748                .state
1749                .judgements
1750                .get(j)
1751                .and_then(|r| r.ranking.first().copied());
1752            let mut record = VoteRecord {
1753                judge: j + 1,
1754                agent: agent_id,
1755                vote: None,
1756                reason: String::new(),
1757                changed: false,
1758            };
1759            match res {
1760                Ok((v, _)) => {
1761                    record.vote = v.label();
1762                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1763                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1764                    self.state.event(
1765                        "vote",
1766                        format!(
1767                            "judge {} voted {}{}",
1768                            j + 1,
1769                            record.vote.unwrap_or('?'),
1770                            if record.changed { " (changed)" } else { "" }
1771                        ),
1772                    );
1773                }
1774                Err(e) => {
1775                    self.state
1776                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1777                }
1778            }
1779            self.state.votes.push(record);
1780            self.state.save()?;
1781        }
1782        Ok(())
1783    }
1784
1785    // --------------------------------------------------------------- tally
1786
1787    fn tally(&mut self) -> Result<()> {
1788        if self.state.tally.is_some() {
1789            return Ok(());
1790        }
1791        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1792        let tops: Vec<char> = self
1793            .state
1794            .judgements
1795            .iter()
1796            .filter_map(|j| j.ranking.first().copied())
1797            .collect();
1798        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1799
1800        // A judge whose private vote failed still counted once, in the initial
1801        // ranking; using it beats discarding a whole seat.
1802        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1803        let mut cast: Vec<char> = Vec::new();
1804        for (i, j) in self.state.judgements.iter().enumerate() {
1805            let vote = self
1806                .state
1807                .votes
1808                .iter()
1809                .find(|v| v.judge == i + 1)
1810                .and_then(|v| v.vote)
1811                .or_else(|| j.ranking.first().copied());
1812            if let Some(v) = vote {
1813                *first_choice.entry(v).or_insert(0) += 1;
1814                cast.push(v);
1815            }
1816        }
1817
1818        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1819        for j in &self.state.judgements {
1820            let n = j.ranking.len();
1821            for (pos, label) in j.ranking.iter().enumerate() {
1822                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1823            }
1824        }
1825
1826        let best = first_choice.values().copied().max().unwrap_or(0);
1827        let mut leaders: Vec<char> = first_choice
1828            .iter()
1829            .filter(|(_, v)| **v == best)
1830            .map(|(k, _)| *k)
1831            .collect();
1832        let mut tie_break = None;
1833        if leaders.len() > 1 {
1834            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1835            let borda_leaders: Vec<char> = leaders
1836                .iter()
1837                .copied()
1838                .filter(|l| borda[l] == top_borda)
1839                .collect();
1840            tie_break = Some(if borda_leaders.len() == 1 {
1841                format!(
1842                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1843                    leaders.len()
1844                )
1845            } else {
1846                format!(
1847                    "{} way tie on both first-choice votes and Borda points, broken by label order",
1848                    leaders.len()
1849                )
1850            });
1851            leaders = borda_leaders;
1852            leaders.sort_unstable();
1853        }
1854        let winner = *leaders
1855            .first()
1856            .or(viable.first())
1857            .context("no candidate to declare a winner from")?;
1858
1859        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1860        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1861        let deliberated = !self.state.deliberation.is_empty();
1862
1863        // Whose verdict is this? A rate-limited seat is absent even if it
1864        // ranked before the limit hit, so presence is measured against the
1865        // recorded losses, not just "did a ranking ever appear".
1866        let quota_seats: std::collections::BTreeSet<&str> =
1867            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1868        let mut present = 0usize;
1869        for (i, j) in self.state.judgements.iter().enumerate() {
1870            if quota_seats.contains(j.seat.as_str()) {
1871                continue;
1872            }
1873            let ranked = !j.ranking.is_empty() && j.failed.is_none();
1874            let voted = self
1875                .state
1876                .votes
1877                .iter()
1878                .any(|v| v.judge == i + 1 && v.vote.is_some());
1879            if ranked || voted {
1880                present += 1;
1881            }
1882        }
1883        // Strict majority of the configured panel. A bare majority is real
1884        // signal we can act on, while a minority verdict must never stand in
1885        // for a healthy one. A one-candidate run needs no panel at all, and
1886        // `judges` stays `0` rather than the roster size a panel that never
1887        // sat would otherwise be credited with.
1888        let needs_quorum = viable.len() > 1;
1889        let judges_total = if needs_quorum {
1890            self.roles.judges.len()
1891        } else {
1892            0
1893        };
1894        let quorum = if needs_quorum {
1895            judges_total / 2 + 1
1896        } else {
1897            0
1898        };
1899        let met_quorum = !needs_quorum || present >= quorum;
1900        let uncontested = (!needs_quorum).then(|| {
1901            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1902        });
1903
1904        self.state.event(
1905            "tally",
1906            match &uncontested {
1907                Some(reason) => format!("winner {winner} — {reason}"),
1908                None => format!(
1909                    "winner {winner} — votes {} | initial {} | {} changed | \
1910                     {present}/{judges_total} judges{}",
1911                    first_choice
1912                        .iter()
1913                        .map(|(k, v)| format!("{k}:{v}"))
1914                        .collect::<Vec<_>>()
1915                        .join(" "),
1916                    if unanimous_initial {
1917                        "unanimous"
1918                    } else {
1919                        "split"
1920                    },
1921                    changed_votes,
1922                    if met_quorum {
1923                        String::new()
1924                    } else {
1925                        format!(" — below quorum ({quorum} required)")
1926                    },
1927                ),
1928            },
1929        );
1930        if !met_quorum {
1931            self.state.event(
1932                "stall",
1933                format!(
1934                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1935                     the run stops here, resumable"
1936                ),
1937            );
1938        }
1939        self.state.tally = Some(Tally {
1940            first_choice,
1941            borda,
1942            winner,
1943            rankings: tops.len(),
1944            unanimous_initial,
1945            deliberated,
1946            changed_votes,
1947            unanimous_final,
1948            tie_break,
1949            judges: judges_total,
1950            present,
1951            quorum,
1952            met_quorum,
1953            uncontested,
1954        });
1955        self.state.status = if met_quorum {
1956            RunStatus::Reviewing
1957        } else {
1958            RunStatus::Stalled
1959        };
1960        self.state.save()?;
1961        Ok(())
1962    }
1963
1964    // ------------------------------------------------------------- recover
1965
1966    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
1967    /// resumed toward completion once the transient cause clears.
1968    ///
1969    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
1970    /// it toward the quorum, which is exactly the set of seats whose absence
1971    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
1972    /// not depend on which node happened to hit the limit), or an ordinary
1973    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
1974    /// seat is never disturbed.
1975    ///
1976    /// A seat that now answers with a usable ranking is "recovered": its
1977    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
1978    /// `tally` counts it present again), and its vote re-collected. A seat that
1979    /// still fails keeps its loss and stays absent.
1980    ///
1981    /// Returns `true` when the re-tally restores the quorum (the run may proceed
1982    /// to review/gate/merge), `false` when it is still below quorum (the run
1983    /// stays `Stalled`, still resumable for a later retry).
1984    #[allow(clippy::too_many_lines)]
1985    async fn recover_stall(&mut self) -> Result<bool> {
1986        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1987        // agent files with `magi task add` name the run that paid for it. The
1988        // prompt overlay is cloned alongside it because the waves borrow it
1989        // while `self` is mutably borrowed by the node's own bookkeeping.
1990        let run_id = self.state.id.clone();
1991        let prompts = self.state.config.prompts.clone();
1992        // Absent seats = quota-lost at any node, or failed outright. Mirroring
1993        // `tally`'s presence test (rather than the old quota-judge/vote filter)
1994        // is what keeps a non-quota collapse — or a quota loss recorded at the
1995        // deliberate node — from being a permanent dead-end on `--resume`.
1996        let quota_seats: BTreeSet<&str> =
1997            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1998        let absent: Vec<String> = self
1999            .state
2000            .judgements
2001            .iter()
2002            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
2003            .map(|j| j.seat.clone())
2004            .collect();
2005        if absent.is_empty() {
2006            return Ok(false);
2007        }
2008        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
2009        if viable.len() <= 1 {
2010            return Ok(false);
2011        }
2012        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
2013        let language = self.state.config.graph.language.clone();
2014        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
2015        let sessions = self.state.config.graph.sessions;
2016        let artifacts = agent::artifacts_dir(&self.state.dir());
2017        let root = self.state.worktree_root();
2018        let base_short = short(&self.state.base_commit);
2019        let candidates: Vec<Candidate> = viable.clone();
2020
2021        // Map each absent seat key to its 0-based position in `roles.judges`.
2022        let mut positions: Vec<usize> = absent
2023            .iter()
2024            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
2025            .collect();
2026        if positions.is_empty() {
2027            return Ok(false);
2028        }
2029        positions.sort_unstable();
2030        positions.dedup();
2031
2032        // Re-rank the lost seats, one blind prompt each.
2033        let mut judge_jobs = Vec::new();
2034        for &j in &positions {
2035            let order = blind::presentation_order(viable.len(), j, self.state.seed);
2036            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
2037            let seat_key = format!("judge-{}", j + 1);
2038            let spec = self.roles.judges[j].clone();
2039            let seat = self.seat(&seat_key, &spec.id);
2040            judge_jobs.push(SeatJob {
2041                spec,
2042                seat,
2043                prompt: prompt::judge(
2044                    &self.state.instruction,
2045                    &views,
2046                    self.roles.judges.len(),
2047                    &base_short,
2048                    &language,
2049                ),
2050                cwd: root.join(seat_key),
2051                timeout,
2052                allow_write: false,
2053                sessions,
2054                artifacts: artifacts.clone(),
2055                stem: format!("judge-{}-recover", j + 1),
2056            });
2057        }
2058
2059        let labels_for_check = labels.clone();
2060        let mut judge_losses = Vec::new();
2061        let retries = self.state.config.graph.retries;
2062        let cache = self.state.config.cache_dir();
2063        let ctx = WaveCtx {
2064            run: &run_id,
2065            node: "judge",
2066            prompts: &prompts,
2067            cache: cache.as_deref(),
2068        };
2069        let results = ask_json_wave::<Ranking>(
2070            judge_jobs,
2071            Arc::clone(&self.sem),
2072            retries,
2073            &ctx,
2074            &mut judge_losses,
2075            &mut self.state,
2076            &move |r: &Ranking| r.validate(&labels_for_check),
2077        )
2078        .await;
2079
2080        // Refresh the judgement of every seat that ranked again.
2081        let mut recovered: BTreeSet<usize> = BTreeSet::new();
2082        for (&j, (seat, res)) in positions.iter().zip(results) {
2083            self.state.seats.insert(seat.key.clone(), seat);
2084            let record = &mut self.state.judgements[j];
2085            match res {
2086                Ok((ranking, out)) => {
2087                    record.ranking = ranking.normalized();
2088                    record.reasons = ranking.reasons;
2089                    record.confidence = ranking.confidence;
2090                    record.failed = None;
2091                    record.duration_ms = out.duration_ms;
2092                    recovered.insert(j);
2093                    self.state.event(
2094                        "recover",
2095                        format!("judge {} ranked again after the limit", j + 1),
2096                    );
2097                }
2098                Err(e) => {
2099                    self.state
2100                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
2101                }
2102            }
2103        }
2104
2105        // Re-ask the votes of the seats that recovered a ranking.
2106        let mut vote_jobs = Vec::new();
2107        let mut vote_pos: Vec<usize> = Vec::new();
2108        for &j in &recovered {
2109            let seat_key = format!("judge-{}", j + 1);
2110            let spec = self.roles.judges[j].clone();
2111            let seat = self.seat(&seat_key, &spec.id);
2112            let mut text = prompt::final_vote(&labels, &language);
2113            if !has_context(&spec, &seat, sessions) {
2114                text = format!(
2115                    "{}\n\n# Candidates\n\n{}",
2116                    text,
2117                    self.candidate_block(&candidates, &base_short)
2118                );
2119            }
2120            vote_jobs.push(SeatJob {
2121                spec,
2122                seat,
2123                prompt: text,
2124                cwd: root.join(seat_key),
2125                timeout,
2126                allow_write: false,
2127                sessions,
2128                artifacts: artifacts.clone(),
2129                stem: format!("vote-judge-{}-recover", j + 1),
2130            });
2131            vote_pos.push(j);
2132        }
2133        let allowed = labels.clone();
2134        let mut vote_losses = Vec::new();
2135        let vote_retries = self.state.config.graph.retries;
2136        let vote_cache = self.state.config.cache_dir();
2137        let ctx = WaveCtx {
2138            run: &run_id,
2139            node: "vote",
2140            prompts: &prompts,
2141            cache: vote_cache.as_deref(),
2142        };
2143        let votes = ask_json_wave::<FinalVote>(
2144            vote_jobs,
2145            Arc::clone(&self.sem),
2146            vote_retries,
2147            &ctx,
2148            &mut vote_losses,
2149            &mut self.state,
2150            &move |v: &FinalVote| match v.label() {
2151                Some(c) if allowed.contains(&c) => Ok(()),
2152                other => bail!("vote {other:?} is not one of {allowed:?}"),
2153            },
2154        )
2155        .await;
2156        for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
2157            let agent_id = seat.agent.clone();
2158            self.state.seats.insert(seat.key.clone(), seat);
2159            match res {
2160                Ok((v, _)) => {
2161                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
2162                        rec.vote = v.label();
2163                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
2164                    } else {
2165                        self.state.votes.push(VoteRecord {
2166                            judge: j + 1,
2167                            agent: agent_id,
2168                            vote: v.label(),
2169                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
2170                            changed: false,
2171                        });
2172                    }
2173                    self.state.event(
2174                        "recover",
2175                        format!("judge {} voted again after the limit", j + 1),
2176                    );
2177                }
2178                Err(e) => {
2179                    self.state
2180                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
2181                }
2182            }
2183        }
2184
2185        // A seat that ranked again is present even if its re-vote failed —
2186        // `tally` falls back to the initial ranking's first choice — so clear
2187        // its quota loss. Seats that still fail keep theirs and stay absent.
2188        if !recovered.is_empty() {
2189            let recovered_keys: BTreeSet<String> = recovered
2190                .iter()
2191                .map(|&j| format!("judge-{}", j + 1))
2192                .collect();
2193            self.state
2194                .quota
2195                .retain(|q| !recovered_keys.contains(&q.seat));
2196        }
2197
2198        // Recompute the verdict from the refreshed panel.
2199        self.state.tally = None;
2200        self.tally()?;
2201        Ok(self
2202            .state
2203            .tally
2204            .as_ref()
2205            .map(|t| t.met_quorum)
2206            .unwrap_or(false))
2207    }
2208
2209    // ----------------------------------------------------------------- fold
2210
2211    async fn fold_losers(&mut self) -> Result<()> {
2212        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
2213            return Ok(());
2214        };
2215        let repo = self.state.repo.clone();
2216        let mut folded = Vec::new();
2217        for i in 0..self.state.candidates.len() {
2218            let c = &self.state.candidates[i];
2219            if c.label == winner || c.folded {
2220                continue;
2221            }
2222            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
2223            git::worktree_remove(&repo, &wt).await.ok();
2224            git::branch_delete(&repo, &branch).await.ok();
2225            self.state.candidates[i].folded = true;
2226            folded.push(label.to_string());
2227        }
2228        // The judges are finished; their checkouts are pure cost from here.
2229        let root = self.state.worktree_root();
2230        for j in 1..=self.roles.judges.len() {
2231            let wt = root.join(format!("judge-{j}"));
2232            if wt.exists() {
2233                git::worktree_remove(&repo, &wt).await.ok();
2234            }
2235        }
2236        // The design-deliberation stage is finished by the time a tally
2237        // exists — same reasoning as the judges above.
2238        if self.state.config.graph.advise {
2239            for k in 1..=self.state.config.graph.advisors {
2240                let wt = root.join(format!("advisor-{k}"));
2241                if wt.exists() {
2242                    git::worktree_remove(&repo, &wt).await.ok();
2243                }
2244            }
2245        }
2246        if !folded.is_empty() {
2247            self.state
2248                .event("fold", format!("folded candidates {}", folded.join(", ")));
2249            self.state.save()?;
2250        }
2251        Ok(())
2252    }
2253
2254    // ------------------------------------------------------------ base sync
2255
2256    /// Land the winner's tree on the current tip of `<remote>/<base>` before
2257    /// anything verifies it.
2258    ///
2259    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
2260    /// read whatever is checked out in the winner's worktree. Left alone that
2261    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
2262    /// it when the run *branched* - and a run takes long enough that the base
2263    /// has usually moved by the time it gets here. A gate that ran there
2264    /// answers "green on the commit this run started from", not "green on
2265    /// what is about to land", and the difference showed up three times in
2266    /// one day as a green run whose merge would have reverted a file another
2267    /// pull request had already landed.
2268    ///
2269    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
2270    /// implementation of the same idea: `land::Step::Rebase` already worked
2271    /// out the rules - throwaway worktree, conflict stops and reports rather
2272    /// than feeding a fixer, nothing runs in the primary tree - and a second
2273    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
2274    /// warns about ("two answers to a question nobody notices until a diff is
2275    /// wrong").
2276    ///
2277    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
2278    /// so it survives a park/resume. A conflict or a push failure sets
2279    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
2280    /// as they were - untouched, for a person to look at - which is also what
2281    /// makes re-entering this function afterwards a no-op instead of a second
2282    /// attempt at the same wall.
2283    async fn sync_to_base(&mut self) -> Result<()> {
2284        if self
2285            .state
2286            .base_sync
2287            .as_ref()
2288            .is_some_and(|s| s.conflict.is_some())
2289        {
2290            return Ok(());
2291        }
2292        let Some(winner) = self.state.winner().cloned() else {
2293            return Ok(());
2294        };
2295
2296        let repo = self.state.repo.clone();
2297        let remote = self.state.config.merge.remote.clone();
2298        let base_branch = self.state.base_branch.clone();
2299        let tracking = format!("{remote}/{base_branch}");
2300
2301        git::fetch(&repo, &remote, &base_branch).await.ok();
2302        // No network, or the remote never had this branch: `resolve_base`
2303        // already treats that as non-fatal at branch time, and a run that got
2304        // this far must not be blocked by it here either.
2305        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
2306            return Ok(());
2307        };
2308
2309        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2310        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
2311        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
2312
2313        if behind == 0 {
2314            self.state.base_sync = Some(BaseSync {
2315                tip,
2316                behind: 0,
2317                attempts,
2318                conflict: None,
2319            });
2320            self.state.save()?;
2321            return Ok(());
2322        }
2323
2324        if attempts >= BASE_SYNC_ROUNDS {
2325            let why = format!(
2326                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
2327                 rebase(s); rebasing again would only race it",
2328                winner.branch
2329            );
2330            self.state.status = RunStatus::Blocked;
2331            self.state.base_sync = Some(BaseSync {
2332                tip,
2333                behind,
2334                attempts,
2335                conflict: Some(why.clone()),
2336            });
2337            self.state.event("land", why);
2338            self.state.save()?;
2339            return Ok(());
2340        }
2341
2342        self.state.event(
2343            "land",
2344            format!(
2345                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
2346                winner.branch
2347            ),
2348        );
2349        self.state.save()?;
2350
2351        let scratch = self.state.dir().join("base-sync");
2352        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
2353        let attempts = attempts + 1;
2354        match rebased {
2355            Ok(None) => {
2356                // The branch ref moved, but a worktree that already had it
2357                // checked out (the winner's) was not told; sync its index and
2358                // files before anything reads them.
2359                git::sync_to_head(&winner.worktree).await?;
2360                self.state.base_sync = Some(BaseSync {
2361                    tip: tip.clone(),
2362                    behind: 0,
2363                    attempts,
2364                    conflict: None,
2365                });
2366                self.state
2367                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
2368            }
2369            Ok(Some(conflict)) => {
2370                let why = format!(
2371                    "{} conflicts with {tracking} and did not rebase: {}",
2372                    winner.branch,
2373                    conflict.chars().take(600).collect::<String>()
2374                );
2375                self.state.status = RunStatus::Blocked;
2376                self.state.base_sync = Some(BaseSync {
2377                    tip,
2378                    behind,
2379                    attempts,
2380                    conflict: Some(why.clone()),
2381                });
2382                self.state.event("land", why);
2383            }
2384            Err(e) => {
2385                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
2386                self.state.status = RunStatus::Blocked;
2387                self.state.base_sync = Some(BaseSync {
2388                    tip,
2389                    behind,
2390                    attempts,
2391                    conflict: Some(why.clone()),
2392                });
2393                self.state.event("land", why);
2394            }
2395        }
2396        self.state.save()?;
2397        Ok(())
2398    }
2399
2400    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
2401    /// last landed the winner on, once it has run, else the commit the run
2402    /// branched from.
2403    ///
2404    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
2405    /// and `vote` all happen before there is a winner to rebase, so they
2406    /// compare every candidate against the branch point on purpose, and a
2407    /// base that moves after they are already done cannot change an answer
2408    /// they already gave.
2409    fn landing_base(&self) -> String {
2410        self.state
2411            .base_sync
2412            .as_ref()
2413            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2414    }
2415
2416    // --------------------------------------------------------------- review
2417
2418    async fn review_loop(&mut self) -> Result<()> {
2419        // A base that would not rebase is a person's decision, not a review
2420        // round: nothing here would change the answer, and reviewers and a
2421        // fixer would be spending real budget on a tree that cannot land
2422        // regardless of what they find.
2423        if self
2424            .state
2425            .base_sync
2426            .as_ref()
2427            .is_some_and(|s| s.conflict.is_some())
2428        {
2429            return Ok(());
2430        }
2431        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2432        // agent files with `magi task add` name the run that paid for it. The
2433        // prompt overlay is cloned alongside it because the waves borrow it
2434        // while `self` is mutably borrowed by the node's own bookkeeping.
2435        let run_id = self.state.id.clone();
2436        let prompts = self.state.config.prompts.clone();
2437        let Some(winner) = self.state.winner().cloned() else {
2438            return Ok(());
2439        };
2440        let max_rounds = self.state.config.graph.review_rounds;
2441        // A clean round, an exhausted round budget, or a stalled tree (see
2442        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
2443        // they are recorded — recomputed here, not read off `status`, so a
2444        // reentry into a run that already stopped restates the identical
2445        // verdict instead of silently handing back whatever an earlier node
2446        // in this same walk clobbered `status` to (a solo-candidate
2447        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
2448        // below runs an empty range once the budget is spent, and would
2449        // otherwise fall through without touching `status` at all.
2450        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
2451            self.state.status = status;
2452            self.state.save()?;
2453            return Ok(());
2454        }
2455        self.state.status = RunStatus::Reviewing;
2456
2457        let repo = self.state.repo.clone();
2458        let root = self.state.worktree_root();
2459        let language = self.state.config.graph.language.clone();
2460        let sessions = self.state.config.graph.sessions;
2461        let artifacts = agent::artifacts_dir(&self.state.dir());
2462        let base = self.landing_base();
2463        let base_short = short(&base);
2464        let reviewers = self.roles.reviewers.clone();
2465        let shell = self.state.config.shell();
2466
2467        let mut prev_e2e: Option<String> = None;
2468        for round in (self.state.reviews.len() + 1)..=max_rounds {
2469            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2470            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2471            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2472
2473            // Each reviewer gets its own detached checkout of exactly this
2474            // commit: nobody can perturb the winner's tree, and the fixer can
2475            // keep working without racing a reviewer.
2476            let mut jobs = Vec::new();
2477            for (r, spec) in reviewers.iter().cloned().enumerate() {
2478                let wt = root.join(format!("review-{}", r + 1));
2479                if wt.exists() {
2480                    git::reset_detached(&wt, &head).await?;
2481                } else {
2482                    git::worktree_add_detached(&repo, &wt, &head).await?;
2483                }
2484                let seat_key = format!("review-{}", r + 1);
2485                let seat = self.seat(&seat_key, &spec.id);
2486                jobs.push(SeatJob {
2487                    prompt: prompt::review(&prompt::ReviewCtx {
2488                        instruction: &self.state.instruction,
2489                        branch: &winner.branch,
2490                        base_short: &base_short,
2491                        stat: &stat,
2492                        patch: &patch,
2493                        e2e: prev_e2e.as_deref(),
2494                        reviewers: reviewers.len(),
2495                        round,
2496                        rounds: max_rounds,
2497                        // A review-only run has no rankings, so nothing
2498                        // competed for this patch and the reviewer is told so.
2499                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2500                        lens: Lens::for_seat(r),
2501                        language: &language,
2502                    }),
2503                    spec,
2504                    seat,
2505                    cwd: wt,
2506                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2507                    allow_write: false,
2508                    sessions,
2509                    artifacts: artifacts.clone(),
2510                    stem: format!("review-{round}-{}", r + 1),
2511                });
2512            }
2513
2514            self.state.event(
2515                "review",
2516                format!(
2517                    "round {round}: {} reviewers on {}",
2518                    jobs.len(),
2519                    short(&head)
2520                ),
2521            );
2522            let mut quota_losses = Vec::new();
2523            let review_retries = self.state.config.graph.retries;
2524            let review_cache = self.state.config.cache_dir();
2525            let ctx = WaveCtx {
2526                run: &run_id,
2527                node: "review",
2528                prompts: &prompts,
2529                cache: review_cache.as_deref(),
2530            };
2531            let results = ask_json_wave::<Review>(
2532                jobs,
2533                Arc::clone(&self.sem),
2534                review_retries,
2535                &ctx,
2536                &mut quota_losses,
2537                &mut self.state,
2538                &|_: &Review| Ok(()),
2539            )
2540            .await;
2541            // Counted before the move below: how many of *this* round's
2542            // reviewer seats were lost to their own rate limit, as opposed to
2543            // a crash, a timeout, or unparsable output — see `round_is_clean`.
2544            let round_quota_missing = quota_losses.len();
2545            self.state.quota.extend(quota_losses);
2546
2547            let mut records = Vec::new();
2548            let mut all_findings = Vec::new();
2549            for (r, (seat, res)) in results.into_iter().enumerate() {
2550                let agent_id = seat.agent.clone();
2551                self.state.seats.insert(seat.key.clone(), seat);
2552                let mut record = ReviewRecord {
2553                    reviewer: r + 1,
2554                    agent: agent_id,
2555                    summary: String::new(),
2556                    findings: Vec::new(),
2557                    vote: None,
2558                    failed: None,
2559                    duration_ms: 0,
2560                };
2561                match res {
2562                    Ok((review, out)) => {
2563                        // Sanitized here, at the point every other piece of
2564                        // agent prose in this file is (candidate summaries,
2565                        // deliberation turns, vote reasons): a reviewer's own
2566                        // words are the one thing about it that could name
2567                        // it, and reconsideration below broadcasts this same
2568                        // summary and these same findings to every other
2569                        // seat on the panel.
2570                        record.summary =
2571                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
2572                        record.vote = Some(review.vote);
2573                        record.duration_ms = out.duration_ms;
2574                        for (n, mut f) in review.findings.into_iter().enumerate() {
2575                            // ids are magi's, never the agent's: the fixer's
2576                            // adoption report is keyed by them.
2577                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2578                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
2579                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
2580                            // `file` is agent-supplied prose too, never
2581                            // checked against the real tree — the same
2582                            // exposure `title`/`detail` above have, just in
2583                            // a field easy to forget because it looks like a
2584                            // path rather than free text.
2585                            f.file = f
2586                                .file
2587                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
2588                            all_findings.push(f.clone());
2589                            record.findings.push(f);
2590                        }
2591                        self.state.event(
2592                            "review",
2593                            format!(
2594                                "round {round}: reviewer {} voted {} with {} finding(s)",
2595                                r + 1,
2596                                review.vote.label(),
2597                                record.findings.len()
2598                            ),
2599                        );
2600                    }
2601                    Err(e) => {
2602                        record.failed = Some(e.to_string());
2603                        self.state.event(
2604                            "review",
2605                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2606                        );
2607                    }
2608                }
2609                records.push(record);
2610            }
2611
2612            // Tally the round's votes and, if they split, spend the one
2613            // round of reconsideration the split -> deliberate -> revote
2614            // shape `judge`/`vote` use for the panel, sized down to what a
2615            // read-only review round can afford: one round, and a revote
2616            // rather than an argument, because the panel already wrote its
2617            // reasoning down as findings the first time around.
2618            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
2619            let vote_split =
2620                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
2621            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
2622            if vote_split {
2623                self.state.event(
2624                    "review",
2625                    format!(
2626                        "round {round}: votes split ({}) — one round of reconsideration",
2627                        initial_votes
2628                            .iter()
2629                            .map(|v| v.label())
2630                            .collect::<Vec<_>>()
2631                            .join(", ")
2632                    ),
2633                );
2634                // Seats read every seat's findings and votes, still numbered
2635                // and never named — the same anonymity `review` itself keeps.
2636                let panel: Vec<ReviewSeatReport<'_>> = records
2637                    .iter()
2638                    .filter_map(|r| {
2639                        r.vote.map(|vote| ReviewSeatReport {
2640                            reviewer: r.reviewer,
2641                            vote,
2642                            summary: &r.summary,
2643                            findings: &r.findings,
2644                        })
2645                    })
2646                    .collect();
2647
2648                let mut jobs = Vec::new();
2649                let mut seats_at = Vec::new();
2650                for (r, spec) in reviewers.iter().cloned().enumerate() {
2651                    // A seat with no initial vote has nothing to reconsider
2652                    // from and stays absent, the same as it stayed absent
2653                    // from `panel` above.
2654                    if records[r].vote.is_none() {
2655                        continue;
2656                    }
2657                    let wt = root.join(format!("review-{}", r + 1));
2658                    let seat_key = format!("review-{}", r + 1);
2659                    let seat = self.seat(&seat_key, &spec.id);
2660                    // A seat with no live session has already forgotten the
2661                    // initial review's prompt — restate the patch it is
2662                    // voting on, the same as `deliberate`/`vote` do for a
2663                    // judge in the same position.
2664                    let patch_ctx = if has_context(&spec, &seat, sessions) {
2665                        None
2666                    } else {
2667                        Some(ReviewPatch {
2668                            branch: &winner.branch,
2669                            base_short: &base_short,
2670                            stat: &stat,
2671                            patch: &patch,
2672                        })
2673                    };
2674                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
2675                        instruction: &self.state.instruction,
2676                        reviewer: r + 1,
2677                        lens: Lens::for_seat(r),
2678                        panel: &panel,
2679                        patch: patch_ctx,
2680                        round,
2681                        rounds: max_rounds,
2682                        language: &language,
2683                    });
2684                    jobs.push(SeatJob {
2685                        prompt,
2686                        spec,
2687                        seat,
2688                        cwd: wt,
2689                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2690                        allow_write: false,
2691                        sessions,
2692                        artifacts: artifacts.clone(),
2693                        stem: format!("review-{round}-reconsider-{}", r + 1),
2694                    });
2695                    seats_at.push(r);
2696                }
2697
2698                let mut recon_quota_losses = Vec::new();
2699                let recon_cache = self.state.config.cache_dir();
2700                let recon_ctx = WaveCtx {
2701                    run: &run_id,
2702                    node: "review",
2703                    prompts: &prompts,
2704                    cache: recon_cache.as_deref(),
2705                };
2706                let recon_results = ask_json_wave::<ReviewRevote>(
2707                    jobs,
2708                    Arc::clone(&self.sem),
2709                    review_retries,
2710                    &recon_ctx,
2711                    &mut recon_quota_losses,
2712                    &mut self.state,
2713                    &|_: &ReviewRevote| Ok(()),
2714                )
2715                .await;
2716                self.state.quota.extend(recon_quota_losses);
2717
2718                for (&r, (seat, res)) in seats_at.iter().zip(recon_results) {
2719                    let agent_id = seat.agent.clone();
2720                    self.state.seats.insert(seat.key.clone(), seat);
2721                    let mut rec = ReviewRevoteRecord {
2722                        reviewer: r + 1,
2723                        agent: agent_id,
2724                        vote: None,
2725                        reason: String::new(),
2726                        failed: None,
2727                    };
2728                    match res {
2729                        Ok((rv, _)) => {
2730                            rec.vote = Some(rv.vote);
2731                            rec.reason =
2732                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
2733                            self.state.event(
2734                                "review",
2735                                format!(
2736                                    "round {round}: reviewer {} revoted {}",
2737                                    r + 1,
2738                                    rv.vote.label()
2739                                ),
2740                            );
2741                        }
2742                        Err(e) => {
2743                            rec.failed = Some(e.to_string());
2744                            self.state.event(
2745                                "review",
2746                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
2747                            );
2748                        }
2749                    }
2750                    reconsideration.push(rec);
2751                }
2752            } else if initial_votes.len() > 1 {
2753                self.state.event(
2754                    "review",
2755                    format!(
2756                        "round {round}: votes agreed ({}) — no reconsideration",
2757                        initial_votes[0].label()
2758                    ),
2759                );
2760            }
2761
2762            // The final vote per seat is its revote where reconsideration
2763            // ran and answered, its initial vote otherwise — the same
2764            // fallback `tally` uses for a judge whose private vote failed.
2765            let final_votes: Vec<ReviewVote> = records
2766                .iter()
2767                .filter_map(|r| {
2768                    reconsideration
2769                        .iter()
2770                        .find(|rv| rv.reviewer == r.reviewer)
2771                        .and_then(|rv| rv.vote)
2772                        .or(r.vote)
2773                })
2774                .collect();
2775            let round_verdict = ReviewVote::worst(final_votes);
2776
2777            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2778            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
2779            // A round that already has a blocking finding and a round left to
2780            // try is going back to the fixer no matter what `verify.e2e`
2781            // says, so running it first only spends the loop's slowest step
2782            // (minutes, for a Rust repo's full test suite) on a head about
2783            // to be rewritten. Deferred, never skipped: `verify.e2e` still
2784            // runs once a round has no blocking findings left (see
2785            // `round_is_clean`, which a deferred — empty — `e2e` can never
2786            // satisfy since `blocking` is nonzero whenever this branch is
2787            // taken), and `stop_reviewing` forces a real run before it will
2788            // ever read a deferred round as green.
2789            let defer_e2e =
2790                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
2791            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
2792                let reason =
2793                    format!("{blocking} blocking finding(s) already required a fix this round");
2794                self.state.event(
2795                    "verify",
2796                    format!(
2797                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
2798                         {}); it will run once a round has none left",
2799                        short(&head)
2800                    ),
2801                );
2802                (Vec::new(), false, true, Some(reason))
2803            } else {
2804                let e2e_commands = self.state.config.verify.e2e.clone();
2805                let (e2e, verify_retried) = run_e2e_with_retry(
2806                    &mut self.state,
2807                    &shell,
2808                    &e2e_commands,
2809                    &winner.worktree,
2810                    verify_timeout,
2811                    &format!("round {round}"),
2812                )
2813                .await;
2814                (e2e, verify_retried, false, None)
2815            };
2816
2817            let e2e_failures: String = e2e
2818                .iter()
2819                .filter(|o| !o.ok())
2820                .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2821                .collect();
2822
2823            let expected = records.len();
2824            let answered = records.iter().filter(|r| r.failed.is_none()).count();
2825            let incomplete = answered < expected;
2826            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
2827            let policy = self.state.config.graph.incomplete_review;
2828            let clean = round_is_clean(
2829                blocking,
2830                e2e_ok,
2831                answered,
2832                expected,
2833                round_quota_missing,
2834                policy,
2835            );
2836
2837            let mut round_record = ReviewRound {
2838                round,
2839                head: head.clone(),
2840                verified_head: None,
2841                reviews: records,
2842                e2e,
2843                verify_retried,
2844                e2e_deferred,
2845                e2e_defer_reason,
2846                fix: None,
2847                blocking,
2848                answered,
2849                expected,
2850                clean,
2851                progressed: false,
2852                vote_split,
2853                reconsideration,
2854                verdict: round_verdict,
2855            };
2856
2857            if incomplete {
2858                let missing: Vec<String> = round_record
2859                    .reviews
2860                    .iter()
2861                    .filter(|r| r.failed.is_some())
2862                    .map(|r| format!("review-{}", r.reviewer))
2863                    .collect();
2864                self.state.event(
2865                    "review",
2866                    format!(
2867                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
2868                        missing.join(", ")
2869                    ),
2870                );
2871            }
2872
2873            if clean {
2874                self.state.event(
2875                    "review",
2876                    if incomplete && policy == IncompleteReviewPolicy::Warn {
2877                        format!(
2878                            "round {round}: clean (warn policy, incomplete panel) — no \
2879                             blocking findings from the seats that answered, verification green"
2880                        )
2881                    } else if incomplete {
2882                        format!(
2883                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
2884                             quorum) — no blocking findings from the seats that answered, \
2885                             verification green",
2886                            expected - answered
2887                        )
2888                    } else {
2889                        format!("round {round}: clean — no blocking findings, verification green")
2890                    },
2891                );
2892                self.state.reviews.push(round_record);
2893                self.state.status = RunStatus::Gating;
2894                self.state.save()?;
2895                return Ok(());
2896            }
2897
2898            // Nothing was raised and verification passed, but not every seat
2899            // answered and `round_is_clean` still refused to call it clean —
2900            // either a seat is missing for a reason other than its own quota
2901            // (a crash, a timeout, unparsable output — worth another try), or
2902            // every seat that could have answered lost its quota and nobody
2903            // is left to decide on: re-review rather than send the fixer
2904            // after a round with nothing to fix.
2905            if incomplete && blocking == 0 && e2e_ok {
2906                self.state.reviews.push(round_record);
2907                self.state.save()?;
2908                if round == max_rounds {
2909                    self.state.status = RunStatus::Blocked;
2910                    self.state.event(
2911                        "review",
2912                        format!(
2913                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
2914                             refusing to call it clean",
2915                            expected - answered
2916                        ),
2917                    );
2918                    return Ok(());
2919                }
2920                prev_e2e = None;
2921                continue;
2922            }
2923
2924            if round == max_rounds {
2925                self.state.reviews.push(round_record);
2926                return self
2927                    .stop_reviewing(
2928                        &format!(
2929                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
2930                        ),
2931                        &shell,
2932                        &winner.worktree,
2933                    )
2934                    .await;
2935            }
2936
2937            // Fix. The winner's own implementer seat continues its conversation:
2938            // the competition is over, so context is pure benefit now.
2939            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
2940                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
2941                _ => (
2942                    self.state
2943                        .config
2944                        .agent(&winner.agent)
2945                        .cloned()
2946                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
2947                    format!("impl-{}", winner.label),
2948                ),
2949            };
2950            let seat = self.seat(&fix_seat_key, &fix_spec.id);
2951            let blocking_findings: Vec<_> = all_findings
2952                .iter()
2953                .filter(|f| f.severity.blocks())
2954                .cloned()
2955                .collect();
2956            let job = SeatJob {
2957                prompt: prompt::fix(
2958                    &self.state.instruction,
2959                    &blocking_findings,
2960                    (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
2961                    e2e_deferred,
2962                    round,
2963                    max_rounds,
2964                    &language,
2965                ),
2966                spec: fix_spec.clone(),
2967                seat,
2968                cwd: winner.worktree.clone(),
2969                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
2970                allow_write: true,
2971                sessions,
2972                artifacts: artifacts.clone(),
2973                stem: format!("fix-{round}"),
2974            };
2975            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
2976            let cache = self.state.config.cache_dir();
2977            let ctx = WaveCtx {
2978                run: &run_id,
2979                node: "fix",
2980                prompts: &prompts,
2981                cache: cache.as_deref(),
2982            };
2983            let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2984            let agent_id = seat.agent.clone();
2985            let seat_key = seat.key.clone();
2986            self.state.seats.insert(seat.key.clone(), seat);
2987
2988            let mut fix = FixRecord {
2989                agent: agent_id,
2990                addressed: Vec::new(),
2991                rejected: Vec::new(),
2992                notes: String::new(),
2993                committed: false,
2994                failed: None,
2995                duration_ms: 0,
2996            };
2997            match out {
2998                AgentOutcome::Ok(o) => {
2999                    fix.duration_ms = o.duration_ms;
3000                    match verdict::extract_json::<FixReport>(&o.text) {
3001                        Ok(report) => {
3002                            fix.addressed = report.addressed;
3003                            fix.rejected = report.rejected;
3004                            fix.notes =
3005                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
3006                        }
3007                        Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
3008                    }
3009                }
3010                // The CLI's raw error JSON is not a fix report to parse.
3011                AgentOutcome::Dropped(o) => {
3012                    fix.duration_ms = o.duration_ms;
3013                    let why = o
3014                        .dropped
3015                        .as_ref()
3016                        .map(|d| d.why.as_str())
3017                        .unwrap_or("the CLI ended the stream without delivering its answer");
3018                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
3019                }
3020                AgentOutcome::Quota(o) => {
3021                    self.state.quota.push(QuotaLoss {
3022                        seat: seat_key,
3023                        node: "fix".to_owned(),
3024                        at: Timestamp::now(),
3025                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3026                    });
3027                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
3028                }
3029                AgentOutcome::Failed(e) => fix.failed = Some(e),
3030            }
3031            git::commit_all(
3032                &winner.worktree,
3033                &format!("magi: review round {round} fixes (uncommitted work)"),
3034            )
3035            .await
3036            .ok();
3037            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
3038            fix.committed = after != before;
3039            // Judged by what `git` says moved against base, never by the
3040            // fixer's own `addressed`/`rejected` count — see
3041            // `ReviewRound::progressed`. Propagated with `?`, the same as the
3042            // `patch` snapshot above: swallowing this error would default
3043            // `diff_after` to empty, which almost always differs from a
3044            // non-empty `patch` and reads as "progressed" — exactly backwards
3045            // for a `git` failure the stagnation check cannot see through.
3046            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
3047            let progressed = diff_after != patch;
3048            let commit_note = if fix.committed {
3049                "committed"
3050            } else {
3051                "NO new commit"
3052            };
3053            let tree_note = if progressed {
3054                "changed vs base"
3055            } else {
3056                "unchanged vs base"
3057            };
3058            self.state.event(
3059                "fix",
3060                match &fix.failed {
3061                    // Distinct on purpose from "0 addressed, 0 rejected": the
3062                    // fixer's own diff still landed (blocking counts do keep
3063                    // falling round over round), only its adoption report did
3064                    // not come back, so this must never read like every
3065                    // finding was reviewed and declined.
3066                    Some(reason) => {
3067                        format!(
3068                            "round {round}: fixer's adoption report was lost ({reason}); \
3069                             {commit_note}, tree {tree_note}"
3070                        )
3071                    }
3072                    None => format!(
3073                        "round {round}: {} addressed, {} rejected, {commit_note}, tree {tree_note}",
3074                        fix.addressed.len(),
3075                        fix.rejected.len(),
3076                    ),
3077                },
3078            );
3079            round_record.fix = Some(fix);
3080            round_record.progressed = progressed;
3081            self.state.reviews.push(round_record);
3082            self.state.save()?;
3083
3084            prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
3085
3086            let streak = self
3087                .state
3088                .reviews
3089                .iter()
3090                .rev()
3091                .take_while(|r| !r.progressed)
3092                .count();
3093            if streak >= STAGNANT_LIMIT {
3094                return self
3095                    .stop_reviewing(
3096                        &format!(
3097                            "the tree has not moved against base for {streak} round(s) in a row"
3098                        ),
3099                        &shell,
3100                        &winner.worktree,
3101                    )
3102                    .await;
3103            }
3104        }
3105        Ok(())
3106    }
3107
3108    /// Decide, from the last recorded round's own verification, whether
3109    /// stopping the review loop is a hand-off or a genuine block.
3110    ///
3111    /// Called once the loop has given up trying — the round budget is spent,
3112    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
3113    /// findings still open, never while a round is still clean or the
3114    /// incomplete-panel case handled inline above. Gate and e2e are facts
3115    /// about the tree; a lingering review finding is an opinion, and this
3116    /// workload's own `magi stats` puts reviewer precision low enough
3117    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
3118    /// must not by itself stand between a green, verified change and the
3119    /// human who decides what to do with it. A red e2e is not an opinion, so
3120    /// that case still blocks, with the failing command and a tail of its
3121    /// output recorded here rather than left in `run.json` for someone to go
3122    /// find.
3123    ///
3124    /// A round that deferred its own e2e (see [`Config::graph`]'s
3125    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
3126    /// only because nothing ran, and treating an empty list as a passing one
3127    /// here is exactly the "deferred painted green" bug this function exists
3128    /// to not have. When the last round deferred, this makes the real run —
3129    /// on the actual worktree this loop is about to stop touching — before
3130    /// deciding anything.
3131    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
3132        let round_idx = self.state.reviews.len() - 1;
3133        let needs_catchup_run = {
3134            let last = &self.state.reviews[round_idx];
3135            last.e2e.is_empty() && last.e2e_deferred
3136        };
3137        if needs_catchup_run {
3138            let round = self.state.reviews[round_idx].round;
3139            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
3140            let commands = self.state.config.verify.e2e.clone();
3141            let verified_head = git::rev_parse(worktree, "HEAD").await?;
3142            let (outcomes, verify_retried) = run_e2e_with_retry(
3143                &mut self.state,
3144                shell,
3145                &commands,
3146                worktree,
3147                timeout,
3148                &format!("round {round}: deferred e2e, now catching up before the final decision"),
3149            )
3150            .await;
3151            let last = &mut self.state.reviews[round_idx];
3152            last.e2e = outcomes;
3153            last.verify_retried = verify_retried;
3154            last.e2e_deferred = false;
3155            if verified_head != last.head {
3156                last.verified_head = Some(verified_head);
3157            }
3158        }
3159        let last = &self.state.reviews[round_idx];
3160        let red: Vec<String> = last
3161            .e2e
3162            .iter()
3163            .filter(|o| !o.ok())
3164            .map(|o| {
3165                format!(
3166                    "`{}` -> {:?}\n{}",
3167                    o.command,
3168                    o.code,
3169                    tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3170                )
3171            })
3172            .collect();
3173        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
3174
3175        if red.is_empty() {
3176            self.state.event(
3177                "review",
3178                format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
3179            );
3180            self.state.status = RunStatus::Gating;
3181        } else {
3182            self.state
3183                .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
3184            self.state.status = RunStatus::Blocked;
3185        }
3186        self.state.save()?;
3187        Ok(())
3188    }
3189
3190    // ----------------------------------------------------------------- gate
3191
3192    async fn gate(&mut self) -> Result<()> {
3193        // Judged by the review record itself, not by `status`: a solo
3194        // candidate's `judge`/`deliberate` skip rewrites `status` on every
3195        // reentry (see `judge`), and trusting it here is exactly how a run
3196        // that exhausted its review budget got gated and merged a second
3197        // time around. `review_conclusion` recomputes the review loop's own
3198        // verdict from the round records themselves — `Gating` for a clean
3199        // round or a hand-off (see `stop_reviewing`), anything else means the
3200        // loop is still going or genuinely blocked.
3201        // A base the winner could not be replayed onto is a decision, not a
3202        // round: there is no landing tree to gate. Read as its own record for
3203        // the same reason the review verdict is.
3204        if self.state.status == RunStatus::Failed
3205            || self
3206                .state
3207                .base_sync
3208                .as_ref()
3209                .is_some_and(|s| s.conflict.is_some())
3210            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3211                != Some(RunStatus::Gating)
3212        {
3213            return Ok(());
3214        }
3215        if !self.state.gate.is_empty() {
3216            // `review_loop` derives its conclusion from the clean review
3217            // record on every reentry and therefore puts a completed run back
3218            // in `Gating`. A recorded red gate is a stronger, terminal fact:
3219            // retain its original command output and restore `Blocked` rather
3220            // than pretending the command is still running or running it a
3221            // second time. An empty list remains the only interrupted-gate
3222            // shape that may need to execute a command.
3223            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
3224                self.state.status = RunStatus::Blocked;
3225                self.state.save()?;
3226            }
3227            return Ok(());
3228        }
3229        let Some(winner) = self.state.winner().cloned() else {
3230            return Ok(());
3231        };
3232        self.state.status = RunStatus::Gating;
3233        let shell = self.state.config.shell();
3234        let outcomes = run_commands(
3235            &shell,
3236            &self.state.config.verify.gate,
3237            &winner.worktree,
3238            Duration::from_secs(self.state.config.graph.verify_timeout()),
3239        )
3240        .await;
3241        for o in &outcomes {
3242            self.state.event(
3243                "gate",
3244                format!(
3245                    "`{}` -> {}",
3246                    o.command,
3247                    if o.ok() {
3248                        "pass".to_owned()
3249                    } else {
3250                        format!(
3251                            "FAIL ({:?})\n{}",
3252                            o.code,
3253                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
3254                        )
3255                    }
3256                ),
3257            );
3258        }
3259        let passed = outcomes.iter().all(CommandOutcome::ok);
3260        self.state.gate = outcomes;
3261        if !passed {
3262            self.state.status = RunStatus::Blocked;
3263            self.state.event("gate", "gate failed; not merging");
3264        }
3265        self.state.save()?;
3266        Ok(())
3267    }
3268
3269    // ---------------------------------------------------------------- merge
3270
3271    async fn merge(&mut self) -> Result<()> {
3272        // Same reasoning as `gate`: ask the review and gate records directly
3273        // rather than `status`, which a solo-candidate `judge`/`deliberate`
3274        // skip can rewrite on reentry to something that no longer says
3275        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
3276        // so a hand-off (open findings, green verification) reaches merge
3277        // exactly like a genuinely clean round does.
3278        //
3279        // A run resumed mid-`land` never reaches here at all: `execute`
3280        // recognises `RunStatus::Landing` before it even calls `prep`, and
3281        // routes straight to `run_land` instead. That has to happen a level
3282        // up from this function, not with a check in here, because
3283        // `review_loop`'s own status recomputation (see its doc) runs
3284        // *before* `merge` on every reentry and would otherwise overwrite
3285        // the `Landing` marker with `Gating` before this node ever saw it.
3286        if self
3287            .state
3288            .base_sync
3289            .as_ref()
3290            .is_some_and(|s| s.conflict.is_some())
3291            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
3292                != Some(RunStatus::Gating)
3293            || self.state.gate.iter().any(|o| !o.ok())
3294        {
3295            return Ok(());
3296        }
3297        // This node's own record, not `status`: `status == Ready` is not
3298        // unique to the harmless `MergeMode::None` path this line was
3299        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
3300        // run's PR was closed without merging — and on that run `mode` is
3301        // still `Pr`, so a reentry that fell through here would push and
3302        // open a second pull request. `self.state.merge` is set exactly once
3303        // this node (or `land`) has already produced a verdict, under every
3304        // mode, which is what "already done" actually means here.
3305        if self.state.merge.is_some() {
3306            return Ok(());
3307        }
3308        let Some(winner) = self.state.winner().cloned() else {
3309            return Ok(());
3310        };
3311        let repo = self.state.repo.clone();
3312        let base = self.state.base_branch.clone();
3313        let mode = self.state.config.merge.mode;
3314        let style = self.state.config.merge.style;
3315        let message = pr_body(&self.state, winner.label);
3316
3317        let outcome = match mode {
3318            MergeMode::None => MergeOutcome {
3319                mode,
3320                ok: true,
3321                detail: manual_merge_command(style, &repo, &winner.branch, &message),
3322            },
3323            MergeMode::Local => {
3324                let on = git::current_branch(&repo).await?;
3325                if on.as_deref() != Some(base.as_str()) {
3326                    MergeOutcome {
3327                        mode,
3328                        ok: false,
3329                        detail: format!(
3330                            "{} has {} checked out, not the base branch {base}",
3331                            repo.display(),
3332                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
3333                        ),
3334                    }
3335                } else if !git::is_clean(&repo).await? {
3336                    MergeOutcome {
3337                        mode,
3338                        ok: false,
3339                        detail: format!("{} is dirty; refusing to merge", repo.display()),
3340                    }
3341                } else {
3342                    let out = match style {
3343                        MergeStyle::Merge => {
3344                            git::merge_no_ff(&repo, &winner.branch, &message).await?
3345                        }
3346                        MergeStyle::Squash => {
3347                            git::merge_squash(&repo, &winner.branch, &message).await?
3348                        }
3349                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
3350                    };
3351                    MergeOutcome {
3352                        mode,
3353                        ok: out.ok(),
3354                        detail: if out.ok() { out.stdout } else { out.stderr },
3355                    }
3356                }
3357            }
3358            MergeMode::Pr => {
3359                let remote = self.state.config.merge.remote.clone();
3360                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
3361                if !pushed.ok() {
3362                    MergeOutcome {
3363                        mode,
3364                        ok: false,
3365                        detail: pushed.stderr,
3366                    }
3367                } else {
3368                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
3369                    match out {
3370                        Ok(url) => MergeOutcome {
3371                            mode,
3372                            ok: true,
3373                            detail: url,
3374                        },
3375                        Err(e) => MergeOutcome {
3376                            mode,
3377                            ok: false,
3378                            detail: e.to_string(),
3379                        },
3380                    }
3381                }
3382            }
3383        };
3384
3385        self.state.status = match (mode, outcome.ok) {
3386            (MergeMode::None, _) => RunStatus::Ready,
3387            (_, true) => RunStatus::Merged,
3388            (_, false) => RunStatus::Blocked,
3389        };
3390        self.state.event(
3391            "merge",
3392            format!(
3393                "{:?}: {}",
3394                mode,
3395                outcome.detail.lines().next().unwrap_or("")
3396            ),
3397        );
3398        self.state.merge = Some(outcome);
3399        self.state.save()?;
3400
3401        // The PR is open and the run would historically stop here, leaving the
3402        // operator to watch checks, feed review comments back to a fixer, and
3403        // merge. That was done by hand six times in one session before this
3404        // existed. Opt-in, because merging is the one irreversible thing magi
3405        // can do to a repository.
3406        if self.state.config.graph.land
3407            && mode == MergeMode::Pr
3408            && self.state.status == RunStatus::Merged
3409        {
3410            self.run_land().await?;
3411        }
3412        // `run_land` may have left `status` at `Landing` - still waiting on
3413        // CI or the owner's approval, not actually settled - so this has to
3414        // read whatever `status` ended up as here, not the `Merged` this
3415        // function set a few lines up.
3416        self.settle_questions();
3417        Ok(())
3418    }
3419
3420    /// Enter `land`.
3421    ///
3422    /// Shared between a fresh run's first pass through [`Runner::merge`] and
3423    /// a resumed run's re-entry. `land::land` itself is what serialises the
3424    /// two git-mutating moments inside the loop — the rebase push and
3425    /// `gh pr merge` — per repository (see its own doc); nothing here needs
3426    /// to hold a lock across the whole call, and doing so would serialise
3427    /// this run's CI wait against a *different* run's land-approval resume
3428    /// in the same repository, which is exactly the "must not wait on
3429    /// another task" property the daemon's slot-freeing exists to give.
3430    async fn run_land(&mut self) -> Result<()> {
3431        let url = self
3432            .state
3433            .merge
3434            .as_ref()
3435            .map(|m| m.detail.clone())
3436            .unwrap_or_default();
3437        let url = url.lines().next().unwrap_or("").trim().to_owned();
3438        if !url.starts_with("http") {
3439            return Ok(());
3440        }
3441        // A land failure is not a lost run: the work is on a branch and the
3442        // pull request is open, which is exactly where a human takes over.
3443        match land::land(&mut self.state, &url).await {
3444            Ok(pr) if self.state.parked => {
3445                // `land` already saved the parked marker; nothing here
3446                // overrides `status` back to a terminal value while an
3447                // approval is still outstanding.
3448                let _ = pr;
3449            }
3450            Ok(pr) => {
3451                self.state.status = match pr.state {
3452                    land::PrLifecycle::Merged => RunStatus::Merged,
3453                    _ => RunStatus::Blocked,
3454                };
3455                // Downstream of a confirmed merge only - see
3456                // `bump::should_release_bump`'s own doc for why this one
3457                // check covers all three of `land`'s success paths.
3458                // Best-effort: the run already landed, so a failure here
3459                // (the decision call, `gh`, `cargo`) is recorded and never
3460                // turns a landed run into a failed one.
3461                if bump::should_release_bump(self.state.status)
3462                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
3463                {
3464                    self.state
3465                        .event("bump", format!("release bump skipped: {e:#}"));
3466                }
3467                self.state.save()?;
3468            }
3469            Err(e) => {
3470                self.state.status = RunStatus::Blocked;
3471                self.state.event("land", format!("gave up: {e}"));
3472                self.state.save()?;
3473            }
3474        }
3475        Ok(())
3476    }
3477
3478    // -------------------------------------------------------------- helpers
3479
3480    /// Fetch or create a seat, keeping its conversation across nodes.
3481    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
3482        if let Some(existing) = self.state.seats.get(key)
3483            && existing.agent == agent
3484        {
3485            return existing.clone();
3486        }
3487        let fresh = SeatState::new(key, agent, self.state.seed);
3488        self.state.seats.insert(key.to_owned(), fresh.clone());
3489        fresh
3490    }
3491
3492    /// A candidate rendered for judging, with the leak policy applied.
3493    fn view(&self, c: &Candidate) -> CandidateView {
3494        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3495            .unwrap_or_default();
3496        let (patch, _) = blind::sanitize_patch(
3497            &format!("candidate {} patch", c.label),
3498            &raw,
3499            &self.state.config.blind,
3500        );
3501        CandidateView {
3502            label: c.label,
3503            branch: c.branch.clone(),
3504            summary: c.summary.clone(),
3505            stat: c.stat.clone(),
3506            patch,
3507        }
3508    }
3509
3510    /// The full candidate set as prompt text, for seats with no live session.
3511    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
3512        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
3513        prompt::judge(
3514            "(see above)",
3515            &views,
3516            self.roles.judges.len(),
3517            base_short,
3518            "en",
3519        )
3520    }
3521
3522    /// Anonymised transcript for judge `self_idx`.
3523    ///
3524    /// The initial rankings are always the opening statements. Seeding them
3525    /// only when no turn had been taken yet meant every judge after the first
3526    /// argued against a single voice instead of against the actual split — the
3527    /// disagreement is the information, so it is always on the table.
3528    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
3529        let mut turns = Vec::new();
3530        for j in &self.state.judgements {
3531            if j.ranking.is_empty() {
3532                continue;
3533            }
3534            let reasons = j
3535                .reasons
3536                .iter()
3537                .map(|(k, v)| format!("- {k}: {v}"))
3538                .collect::<Vec<_>>()
3539                .join("\n");
3540            turns.push(Turn {
3541                who: format!("Judge {} (opening ranking)", j.judge),
3542                is_self: j.judge == self_idx + 1,
3543                body: format!(
3544                    "Ranked {}{}{reasons}",
3545                    j.ranking.iter().collect::<String>(),
3546                    if reasons.is_empty() {
3547                        ""
3548                    } else {
3549                        ", because:\n"
3550                    }
3551                ),
3552            });
3553        }
3554        for t in self
3555            .state
3556            .deliberation
3557            .iter()
3558            .flat_map(|r| r.turns.iter())
3559            .chain(current)
3560        {
3561            turns.push(Turn {
3562                who: format!("Judge {}", t.judge),
3563                is_self: t.judge == self_idx + 1,
3564                body: t.body.clone(),
3565            });
3566        }
3567        turns
3568    }
3569}
3570
3571/// Does this seat still hold the context a follow-up prompt would rely on?
3572fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
3573    agent::has_session(spec.kind, seat, sessions)
3574}
3575
3576fn short(commit: &str) -> String {
3577    commit.chars().take(7).collect()
3578}
3579
3580fn make_executable(path: &Path) -> Result<()> {
3581    #[cfg(unix)]
3582    {
3583        use std::os::unix::fs::PermissionsExt as _;
3584        let mut perms = std::fs::metadata(path)?.permissions();
3585        perms.set_mode(0o755);
3586        std::fs::set_permissions(path, perms)?;
3587    }
3588    #[cfg(not(unix))]
3589    {
3590        let _ = path;
3591    }
3592    Ok(())
3593}
3594
3595/// What every seat in one batch shares: where the answers are attributed, the
3596/// prompt overlay they inherit, and the build cache they are told to use.
3597///
3598/// A struct rather than four more parameters: `wave` also needs the run's
3599/// state (to record who is answering right now) and the attempt number, and
3600/// eight positional arguments is both unreadable and a clippy error.
3601struct WaveCtx<'a> {
3602    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
3603    /// paid for it.
3604    run: &'a str,
3605    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
3606    node: &'a str,
3607    prompts: &'a Prompts,
3608    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
3609    cache: Option<&'a Path>,
3610}
3611
3612/// Run one job, honouring the parallelism budget.
3613async fn run_one(
3614    job: SeatJob,
3615    sem: Arc<Semaphore>,
3616    ctx: &WaveCtx<'_>,
3617    state: &mut RunState,
3618    attempt: usize,
3619) -> (SeatState, AgentOutcome) {
3620    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
3621        .await
3622        .pop()
3623        .expect("one job in, one result out");
3624    (seat, out)
3625}
3626
3627/// Run every job concurrently, capped by the semaphore, preserving order.
3628///
3629/// Every seat in the batch is recorded into [`RunState::active`] before the
3630/// wave starts and cleared as each answer lands, so the run's own record says
3631/// who is still being waited on rather than only who finished.
3632async fn wave(
3633    jobs: Vec<SeatJob>,
3634    sem: Arc<Semaphore>,
3635    ctx: &WaveCtx<'_>,
3636    state: &mut RunState,
3637    attempt: usize,
3638) -> Vec<(usize, SeatState, AgentOutcome)> {
3639    let WaveCtx {
3640        run,
3641        node,
3642        prompts,
3643        cache,
3644    } = *ctx;
3645    for job in &jobs {
3646        state.seat_started(node, &job.seat.key, job.timeout, attempt);
3647    }
3648    if let Err(e) = state.save() {
3649        // A failed persist of "who is answering right now" must not abort the
3650        // wave: the seats are already being asked, and the alternative is
3651        // losing the answers to save a status line nobody may even be
3652        // watching.
3653        tracing::warn!("could not persist in-progress seats: {e:#}");
3654    }
3655    let mut set = tokio::task::JoinSet::new();
3656    let overlay = prompts.overlay(node);
3657    for (i, mut job) in jobs.into_iter().enumerate() {
3658        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
3659        if cache.is_some() {
3660            job.prompt.push('\n');
3661            job.prompt.push_str(&prompt::build_cache_note(node));
3662        }
3663        let sem = Arc::clone(&sem);
3664        let run = run.to_owned();
3665        let node = node.to_owned();
3666        let cache = cache.map(Path::to_path_buf);
3667        set.spawn(async move {
3668            let _permit = sem.acquire().await;
3669            let mut seat = job.seat;
3670            let out = agent::invoke(
3671                &job.spec,
3672                &mut seat,
3673                &Invocation {
3674                    cwd: &job.cwd,
3675                    prompt: &job.prompt,
3676                    timeout: job.timeout,
3677                    allow_write: job.allow_write,
3678                    sessions: job.sessions,
3679                    artifacts: &job.artifacts,
3680                    stem: &job.stem,
3681                    run: &run,
3682                    node: &node,
3683                    cache_dir: cache.as_deref(),
3684                    attachments: &[],
3685                },
3686            )
3687            .await;
3688            let out = match out {
3689                Ok(o) if o.usable() => AgentOutcome::Ok(o),
3690                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
3691                // Billed work the CLI failed to hand over is not an ordinary
3692                // failure, but its text is the CLI's raw error JSON, not an
3693                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
3694                // read it as one by forgetting to check. `usable()` is always
3695                // false here (dropped implies an empty response), so this has
3696                // to be checked before the catch-all `Failed` below or the
3697                // one shape this exists for is lost with the rest.
3698                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
3699                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
3700                Ok(o) => AgentOutcome::Failed(format!(
3701                    "exited with {:?} and no usable output",
3702                    o.exit_code
3703                )),
3704                Err(e) => AgentOutcome::Failed(e.to_string()),
3705            };
3706            (i, seat, out)
3707        });
3708    }
3709    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
3710    while let Some(joined) = set.join_next().await {
3711        let (i, seat, out) = match joined {
3712            Ok(v) => v,
3713            // No seat to clear: a panicked task never reported which one it
3714            // was. The defensive sweep below this loop is what stops that
3715            // seat's `active` entry from surviving forever.
3716            Err(e) => {
3717                tracing::error!("agent task panicked: {e}");
3718                continue;
3719            }
3720        };
3721        state.seat_finished(&seat.key);
3722        if let Err(e) = state.save() {
3723            tracing::warn!("could not persist a seat's completion: {e:#}");
3724        }
3725        if collected.len() <= i {
3726            collected.resize_with(i + 1, || None);
3727        }
3728        collected[i] = Some((i, seat, out));
3729    }
3730    // Belt-and-braces for the panic branch above: every seat this exact batch
3731    // started shares this `(node, attempt)` pair, and every seat that finished
3732    // normally already cleared itself, so anything left tagged with it here
3733    // can only be a panicked task's leftover. Cleared unconditionally rather
3734    // than left to read as still answering forever.
3735    if state
3736        .active
3737        .values()
3738        .any(|a| a.node == node && a.attempt == attempt)
3739    {
3740        state
3741            .active
3742            .retain(|_, a| !(a.node == node && a.attempt == attempt));
3743        if let Err(e) = state.save() {
3744            tracing::warn!("could not persist the end of a wave: {e:#}");
3745        }
3746    }
3747    collected.into_iter().flatten().collect()
3748}
3749
3750/// Is a review round clean, given how many reviewer seats answered against
3751/// how many the round expected?
3752///
3753/// A seat that never answered (timeout, crash, unparsable output) is not a
3754/// seat that read the patch and found nothing — treating it as such is
3755/// exactly the bug this function exists to close. Under the default `block`
3756/// policy a missing seat can never be clean; `warn` still requires the seats
3757/// that *did* answer to have found nothing blocking and verification to be
3758/// green.
3759///
3760/// `quota_missing` narrows that `block` default for exactly one cause of
3761/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
3762/// a session limit lifts by the very next round buys nothing — the seat is
3763/// asked again with the same quota — so once every missing seat is accounted
3764/// for by a quota loss (and at least one seat *did* answer, so a decision has
3765/// something to rest on) the round is decided on the panel that could answer,
3766/// same as `warn` would. A panel that lost every seat to quota is not
3767/// decided here: `answered == 0` falls through to the existing `block`
3768/// fallback so a fully collapsed panel still waits rather than landing on no
3769/// review at all.
3770fn round_is_clean(
3771    blocking: usize,
3772    e2e_ok: bool,
3773    answered: usize,
3774    expected: usize,
3775    quota_missing: usize,
3776    policy: IncompleteReviewPolicy,
3777) -> bool {
3778    if blocking != 0 || !e2e_ok {
3779        return false;
3780    }
3781    if answered == expected || policy == IncompleteReviewPolicy::Warn {
3782        return true;
3783    }
3784    answered > 0 && expected - answered <= quota_missing
3785}
3786
3787/// The review loop's own conclusion, derived entirely from its persisted
3788/// round records and the round budget that produced them — never from
3789/// `status`, so a reentry (or `gate`/`merge` reading it independently)
3790/// recomputes the identical answer regardless of what an earlier node in the
3791/// same walk, or a previous walk, did to `status`.
3792///
3793/// `None` while more rounds remain to try, including when review never ran
3794/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
3795/// gone clean, or the budget is spent, or the tree has stopped moving (see
3796/// [`STAGNANT_LIMIT`]), the answer is one of two things:
3797///
3798/// - An incomplete panel that raised nothing is missing input, not a
3799///   verified tree — never a hand-off candidate, whatever verification said
3800///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
3801/// - Otherwise, green e2e on the last round hands off (see
3802///   [`Runner::stop_reviewing`]); red e2e blocks.
3803fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
3804    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
3805        return Some(RunStatus::Gating);
3806    }
3807    let last = reviews.last()?;
3808    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
3809    if reviews.len() < max_rounds && !stagnant {
3810        return None;
3811    }
3812    Some(if last.incomplete() && last.blocking == 0 {
3813        RunStatus::Blocked
3814    } else if last.e2e.iter().all(CommandOutcome::ok) {
3815        RunStatus::Gating
3816    } else {
3817        RunStatus::Blocked
3818    })
3819}
3820
3821/// How long a re-ask may take, given the budget the first attempt had.
3822///
3823/// A `nudged` retry is a request to restate an answer the seat has already
3824/// worked out: it carries no new work, so it does not deserve the original
3825/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
3826/// 133 seconds while a third sat for over ten minutes on a resumed session
3827/// holding 410 KB of prior output - and because the retry had inherited the
3828/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
3829/// judging round whose other seats were long finished.
3830///
3831/// A quarter of the budget, with a floor so that a deliberately short timeout
3832/// does not collapse to nothing. A retry that re-sends the whole prompt
3833/// (because the seat kept no context) is the original job again, and keeps the
3834/// original budget.
3835fn retry_budget(full: Duration, nudged: bool) -> Duration {
3836    if nudged {
3837        (full / 4).max(Duration::from_secs(120)).min(full)
3838    } else {
3839        full
3840    }
3841}
3842
3843/// Run a wave and parse each reply, re-asking the seats whose reply was
3844/// unusable.
3845///
3846/// The re-ask is a nudge rather than the whole prompt again when the seat still
3847/// holds its conversation, which is the difference between a cheap retry and
3848/// paying for the entire candidate set twice.
3849///
3850/// A seat that hits a rate limit is **not** re-asked: the same call will fail
3851/// the same way until the limit resets, so spending a retry attempt on it is
3852/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
3853/// like any other absent seat — the caller decides whether the panel still has
3854/// a quorum.
3855#[allow(clippy::too_many_arguments)]
3856async fn ask_json_wave<T>(
3857    jobs: Vec<SeatJob>,
3858    sem: Arc<Semaphore>,
3859    retries: usize,
3860    ctx: &WaveCtx<'_>,
3861    losses: &mut Vec<QuotaLoss>,
3862    state: &mut RunState,
3863    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
3864) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
3865where
3866    T: serde::de::DeserializeOwned + Send + 'static,
3867{
3868    let n = jobs.len();
3869    let originals: Vec<SeatJob> = jobs;
3870    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
3871    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
3872    let mut pending: Vec<usize> = (0..n).collect();
3873
3874    for attempt in 0..=retries {
3875        if pending.is_empty() {
3876            break;
3877        }
3878        let mut batch = Vec::with_capacity(pending.len());
3879        for &i in &pending {
3880            let src = &originals[i];
3881            // The prompt and the budget are one decision: a nudge restates
3882            // finished work, a re-sent prompt redoes it.
3883            let (prompt, timeout) = if attempt == 0 {
3884                (src.prompt.clone(), src.timeout)
3885            } else {
3886                let why = done[i]
3887                    .as_ref()
3888                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
3889                    .unwrap_or_else(|| "no parsable answer".to_owned());
3890                let nudge = prompt::nudge(&why);
3891                let nudged = has_context(&src.spec, &seats[i], src.sessions);
3892                let prompt = if nudged {
3893                    nudge
3894                } else {
3895                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
3896                };
3897                (prompt, retry_budget(src.timeout, nudged))
3898            };
3899            batch.push(SeatJob {
3900                spec: src.spec.clone(),
3901                seat: seats[i].clone(),
3902                cwd: src.cwd.clone(),
3903                prompt,
3904                timeout,
3905                allow_write: src.allow_write,
3906                sessions: src.sessions,
3907                artifacts: src.artifacts.clone(),
3908                stem: if attempt == 0 {
3909                    src.stem.clone()
3910                } else {
3911                    format!("{}-retry{attempt}", src.stem)
3912                },
3913            });
3914        }
3915
3916        if attempt > 0 {
3917            let seats_out: Vec<&str> = pending
3918                .iter()
3919                .map(|&i| originals[i].seat.key.as_str())
3920                .collect();
3921            state.event(
3922                ctx.node,
3923                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3924            );
3925        }
3926        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3927        let mut still = Vec::new();
3928        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3929            seats[i] = seat;
3930            let (parsed, quota) = match out {
3931                AgentOutcome::Ok(o) => (
3932                    match verdict::extract_json::<T>(&o.text) {
3933                        Ok(v) => match validate(&v) {
3934                            Ok(()) => Ok((v, o)),
3935                            Err(e) => Err(e),
3936                        },
3937                        Err(e) => Err(e),
3938                    },
3939                    false,
3940                ),
3941                AgentOutcome::Quota(o) => {
3942                    losses.push(QuotaLoss {
3943                        seat: originals[i].seat.key.clone(),
3944                        node: ctx.node.to_owned(),
3945                        at: Timestamp::now(),
3946                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3947                    });
3948                    (
3949                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
3950                        true,
3951                    )
3952                }
3953                // Not a parseable answer, but also not worth a special-cased
3954                // retry here: the nudge loop above already re-asks anything
3955                // that fails to parse, which is exactly what a dropped stream
3956                // needs. Just don't hand its raw error JSON to `extract_json`.
3957                AgentOutcome::Dropped(o) => {
3958                    let why = o
3959                        .dropped
3960                        .as_ref()
3961                        .map(|d| d.why.as_str())
3962                        .unwrap_or("the CLI ended the stream without delivering its answer");
3963                    (
3964                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
3965                        false,
3966                    )
3967                }
3968                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
3969            };
3970            let failed = parsed.is_err();
3971            done[i] = Some(parsed);
3972            // Do not re-ask a rate-limited seat (quota) — a retry is known to
3973            // fail the same way; and never re-ask a seat that already parsed.
3974            if failed && !quota {
3975                still.push(i);
3976            }
3977        }
3978        pending = still;
3979    }
3980
3981    seats
3982        .into_iter()
3983        .zip(done)
3984        .map(|(seat, res)| {
3985            (
3986                seat,
3987                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
3988            )
3989        })
3990        .collect()
3991}
3992
3993/// Describe one verify command's outcome for the event log, distinguishing a
3994/// build/link failure — the toolchain never produced a binary to run — from
3995/// an actual test failure, since only the latter is a verdict on the patch.
3996fn e2e_outcome_label(o: &CommandOutcome) -> String {
3997    if o.ok() {
3998        return "pass".to_owned();
3999    }
4000    let reason = if o.build_failed() {
4001        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
4002    } else {
4003        format!("FAIL ({:?})", o.code)
4004    };
4005    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
4006}
4007
4008/// Run `verify.e2e`, retrying once if the first attempt could not build or
4009/// link — a build/link failure is frequently a race against a shared
4010/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
4011/// `verify` event per command, tagged with `context` (normally `"round N"`)
4012/// so the two call sites that need this — the ordinary per-round leg in
4013/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
4014/// it will ever call a round green — read identically in the event log.
4015async fn run_e2e_with_retry(
4016    state: &mut RunState,
4017    shell: &[String],
4018    commands: &[String],
4019    worktree: &Path,
4020    timeout: Duration,
4021    context: &str,
4022) -> (Vec<CommandOutcome>, bool) {
4023    let mut e2e = run_commands(shell, commands, worktree, timeout).await;
4024    for o in &e2e {
4025        state.event(
4026            "verify",
4027            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
4028        );
4029    }
4030    // A build/link failure is not a verdict on the patch — it is frequently a
4031    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
4032    // one retry before letting a red like that decide the round.
4033    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
4034    if verify_retried {
4035        state.event(
4036            "verify",
4037            format!(
4038                "{context}: verify could not build/link, not a test result — retrying once \
4039                 before concluding"
4040            ),
4041        );
4042        e2e = run_commands(shell, commands, worktree, timeout).await;
4043        for o in &e2e {
4044            state.event(
4045                "verify",
4046                format!(
4047                    "{context}: retry `{}` -> {}",
4048                    o.command,
4049                    e2e_outcome_label(o)
4050                ),
4051            );
4052        }
4053    }
4054    (e2e, verify_retried)
4055}
4056
4057/// Run configured shell commands in `cwd`, in order.
4058async fn run_commands(
4059    shell: &[String],
4060    commands: &[String],
4061    cwd: &Path,
4062    timeout: Duration,
4063) -> Vec<CommandOutcome> {
4064    let mut out = Vec::new();
4065    for command in commands {
4066        let started = Instant::now();
4067        let mut cmd = tokio::process::Command::new(&shell[0]);
4068        cmd.quiet();
4069        cmd.args(&shell[1..])
4070            .arg(command)
4071            .current_dir(cwd)
4072            .stdin(std::process::Stdio::null())
4073            .stdout(std::process::Stdio::piped())
4074            .stderr(std::process::Stdio::piped())
4075            .kill_on_drop(true);
4076        let spawned = cmd.spawn();
4077        let (code, body) = match spawned {
4078            Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
4079                Ok(Ok(o)) => {
4080                    let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
4081                    body.push_str(&String::from_utf8_lossy(&o.stderr));
4082                    (o.status.code(), body)
4083                }
4084                Ok(Err(e)) => (None, format!("failed to run: {e}")),
4085                Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
4086            },
4087            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
4088        };
4089        out.push(CommandOutcome {
4090            command: command.clone(),
4091            code,
4092            output_tail: tail(&body, OUTPUT_TAIL),
4093            duration_ms: started.elapsed().as_millis() as u64,
4094        });
4095    }
4096    out
4097}
4098
4099/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
4100/// section (`report::run`) and in the `merge` event this node records — for
4101/// the operator to run by hand.
4102///
4103/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
4104/// branch whose ruleset forbids merge commits (GitHub's "must not contain
4105/// merge commits", or "require linear history") rejects the push a `--no-ff`
4106/// merge would produce, which is exactly the guidance this function replaces.
4107/// `message`'s first line becomes the squash commit's subject, matching the
4108/// note `report::run` prints alongside this command — see that function for
4109/// why an explicit subject is not optional there.
4110fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
4111    let repo = repo.display();
4112    match style {
4113        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
4114        MergeStyle::Squash => {
4115            let subject = message.lines().next().unwrap_or(branch);
4116            format!(
4117                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
4118            )
4119        }
4120        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
4121    }
4122}
4123
4124/// The merge commit / pull request body: the task, and — when the winning
4125/// review round was not clean — the findings still open and whatever the
4126/// fixer declined, so `merge = "pr"` hands the reader the same material
4127/// `magi show` does rather than a pull request that reads clean while
4128/// `run.json` disagrees.
4129///
4130/// The first line doubles as the pull request title (`gh_pr_create`) and the
4131/// squash/merge commit subject (`manual_merge_command`), both of which take
4132/// it via `message.lines().next()` rather than as a separate argument — so it
4133/// has to be the task's own opening line, not run/candidate bookkeeping.
4134/// "Merge magi run ec12 (candidate B)" told a reader nothing about what
4135/// landed once the run id had scrolled off the PR list. That bookkeeping
4136/// still needs to be findable, just not from the title: the branch name
4137/// already carries it (`RunState::branch_for`), and the footer below repeats
4138/// it as plain tags for a reader holding only the merged commit or the PR
4139/// body.
4140///
4141/// `state.instruction` can open with blank lines — a `--file` task is passed
4142/// through verbatim (`task_text` only rejects a body that is blank
4143/// *entirely*) — and `.lines().next()` on those reads back as `Some("")`, not
4144/// `None`, so `gh_pr_create`'s `unwrap_or("magi run")` never fires and `gh pr
4145/// create` would be asked for an empty `--title`. `trim_start` drops exactly
4146/// those leading blank lines so the first line is the task's real opening
4147/// line, and the empty-after-trim case (a whitespace-only instruction) falls
4148/// back the same way `queue::title_from` does for the same situation.
4149fn pr_body(state: &RunState, winner: char) -> String {
4150    let instruction = state.instruction.trim_start();
4151    let mut message = if instruction.is_empty() {
4152        "(empty task)".to_owned()
4153    } else {
4154        instruction.to_owned()
4155    };
4156
4157    let open = state.open_findings();
4158    if !open.is_empty() {
4159        message.push_str("\n\n## Open review findings\n\n");
4160        for f in &open {
4161            message.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
4162        }
4163    }
4164
4165    if let Some(fix) = state.reviews.last().and_then(|r| r.fix.as_ref())
4166        && !fix.rejected.is_empty()
4167    {
4168        message.push_str("\n## Declined by the fixer\n\n");
4169        for r in &fix.rejected {
4170            message.push_str(&format!("- `{}`: {}\n", r.id, r.why));
4171        }
4172    }
4173
4174    message.push_str(&format!(
4175        "\n\n---\nmagi:run/{} magi:candidate-{}\n",
4176        state.id,
4177        winner.to_ascii_lowercase()
4178    ));
4179
4180    message
4181}
4182
4183/// `gh pr create`, returning the PR url.
4184async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
4185    let title = body.lines().next().unwrap_or("magi run").to_owned();
4186    let out = tokio::process::Command::new("gh")
4187        .args([
4188            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
4189        ])
4190        .current_dir(cwd)
4191        .quiet()
4192        .stdin(std::process::Stdio::null())
4193        .output()
4194        .await
4195        .context("spawn gh")?;
4196    if out.status.success() {
4197        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
4198    } else {
4199        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
4200    }
4201}
4202
4203/// Tear a run's worktrees and branches down.
4204pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
4205    let repo = state.repo.clone();
4206    let root = state.worktree_root();
4207    let winner = state.tally.as_ref().map(|t| t.winner);
4208    let mut removed = Vec::new();
4209
4210    for i in 0..state.candidates.len() {
4211        let c = state.candidates[i].clone();
4212        let is_winner = Some(c.label) == winner;
4213        if is_winner && !drop_winner {
4214            continue;
4215        }
4216        if c.worktree.exists() {
4217            git::worktree_remove(&repo, &c.worktree).await.ok();
4218            removed.push(c.worktree.to_string_lossy().into_owned());
4219        }
4220        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
4221            git::branch_delete(&repo, &c.branch).await.ok();
4222            removed.push(c.branch.clone());
4223        }
4224        state.candidates[i].folded = true;
4225    }
4226
4227    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
4228        let path = name.path();
4229        let keep = !drop_winner
4230            && winner.is_some_and(|w| {
4231                path.file_name()
4232                    .is_some_and(|n| n == format!("cand-{w}").as_str())
4233            });
4234        if keep {
4235            continue;
4236        }
4237        git::worktree_remove(&repo, &path).await.ok();
4238        removed.push(path.to_string_lossy().into_owned());
4239    }
4240
4241    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
4242    // judge worktrees, so once the loop above has cleared all of them out,
4243    // the parent is a bare directory nobody else was ever going to remove -
4244    // git only ever managed what was inside it. Left alone, one of these
4245    // accumulates per fully-folded run; the operator's own machine had 74.
4246    // `remove_if_empty` re-checks rather than assuming: a run whose winner
4247    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
4248    // so does anything a run never claimed that happens to share the bay.
4249    remove_if_empty(&root);
4250
4251    if state.enabled_worktree_config && drop_winner {
4252        // A release, not a raw disable: some sibling run in this repository
4253        // may still hold its own reference (see `git::acquire_worktree_config`),
4254        // and only the last release actually turns the setting back off.
4255        git::release_worktree_config(&repo).await.ok();
4256        state.enabled_worktree_config = false;
4257    }
4258    state.save()?;
4259    Ok(removed)
4260}
4261
4262/// Remove `dir` if it exists and has nothing in it.
4263///
4264/// Best-effort and silent by design: a directory that is not empty (a run
4265/// whose winner is still parked there, a stray file some other process left)
4266/// is exactly the case this must refuse, and a directory that is already gone
4267/// is not a failure worth reporting either. `std::fs::remove_dir` itself
4268/// already refuses a non-empty directory, so the emptiness check below is
4269/// belt, not suspenders - it is what keeps this from ever attempting the
4270/// removal in the case that matters, rather than trusting `remove_dir`'s
4271/// error path to have no side effects if it ever changed.
4272fn remove_if_empty(dir: &Path) {
4273    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
4274        std::fs::remove_dir(dir).ok();
4275    }
4276}
4277
4278/// Severity of the worst open finding in the last review round, for reporting.
4279pub fn worst_open(state: &RunState) -> Option<Severity> {
4280    state
4281        .reviews
4282        .last()?
4283        .reviews
4284        .iter()
4285        .flat_map(|r| r.findings.iter())
4286        .map(|f| f.severity)
4287        .max()
4288}
4289
4290#[cfg(test)]
4291mod tests {
4292    use super::*;
4293    use std::collections::BTreeMap;
4294    use std::time::Duration;
4295
4296    fn conductor() -> AgentSpec {
4297        AgentSpec {
4298            id: "conductor".to_owned(),
4299            kind: crate::config::AgentKind::Command,
4300            model: None,
4301            command: vec!["true".to_owned()],
4302            extra_args: Vec::new(),
4303            env: BTreeMap::new(),
4304            prompt_delivery: None,
4305        }
4306    }
4307
4308    #[test]
4309    fn remove_if_empty_only_ever_takes_a_bare_directory() {
4310        let dir = tempfile::tempdir().unwrap();
4311        let bay = dir.path().join("ffff");
4312
4313        // Not there yet: nothing to do, nothing to panic on.
4314        remove_if_empty(&bay);
4315        assert!(!bay.exists());
4316
4317        // Something still inside - the winner's worktree, or a stray file -
4318        // keeps the directory standing.
4319        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
4320        remove_if_empty(&bay);
4321        assert!(bay.exists(), "non-empty directory must survive");
4322
4323        // Once the last entry is gone, so is the directory itself.
4324        std::fs::remove_dir(bay.join("cand-A")).unwrap();
4325        remove_if_empty(&bay);
4326        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
4327    }
4328
4329    // `round_is_clean` is the exact decision this task fixed: a round with a
4330    // seat that never answered must not read the same as a round every seat
4331    // actually reviewed. These are deterministic and process-free by design —
4332    // the equivalent end-to-end check (a real reviewer timing out under a
4333    // live graph run) is a genuine race against wall-clock contention, and a
4334    // spawn slow enough to blow even a generous budget under a loaded test
4335    // run must not turn this specific regression check flaky.
4336
4337    #[test]
4338    fn a_full_panel_that_found_nothing_is_clean() {
4339        assert!(round_is_clean(
4340            0,
4341            true,
4342            2,
4343            2,
4344            0,
4345            IncompleteReviewPolicy::Block
4346        ));
4347    }
4348
4349    #[test]
4350    fn a_missing_seat_is_never_clean_under_the_default_policy() {
4351        assert!(!round_is_clean(
4352            0,
4353            true,
4354            1,
4355            2,
4356            0,
4357            IncompleteReviewPolicy::Block
4358        ));
4359    }
4360
4361    #[test]
4362    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
4363        assert!(!round_is_clean(
4364            1,
4365            true,
4366            1,
4367            2,
4368            0,
4369            IncompleteReviewPolicy::Warn
4370        ));
4371    }
4372
4373    #[test]
4374    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
4375        assert!(round_is_clean(
4376            0,
4377            true,
4378            1,
4379            2,
4380            0,
4381            IncompleteReviewPolicy::Warn
4382        ));
4383    }
4384
4385    #[test]
4386    fn a_full_panel_with_an_open_finding_is_not_clean() {
4387        assert!(!round_is_clean(
4388            1,
4389            true,
4390            2,
4391            2,
4392            0,
4393            IncompleteReviewPolicy::Block
4394        ));
4395    }
4396
4397    #[test]
4398    fn a_full_panel_with_a_red_e2e_is_not_clean() {
4399        assert!(!round_is_clean(
4400            0,
4401            false,
4402            2,
4403            2,
4404            0,
4405            IncompleteReviewPolicy::Block
4406        ));
4407    }
4408
4409    // The stall this task closes: under the default `block` policy, a seat
4410    // missing only because it was rate limited must not force a wait for a
4411    // session limit that will not lift by the next round. `round_is_clean`
4412    // is where that quorum carve-out lives; the review loop around it never
4413    // changes what a reviewer's vote or a finding's severity means.
4414
4415    #[test]
4416    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
4417        // 1 of 2 answered, and the one missing was quota'd — the exact
4418        // "review-2 rate limited (quota)" shape from the field report.
4419        assert!(round_is_clean(
4420            0,
4421            true,
4422            1,
4423            2,
4424            1,
4425            IncompleteReviewPolicy::Block
4426        ));
4427    }
4428
4429    #[test]
4430    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
4431        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
4432        // not a quota loss (`quota_missing` stays 0) — worth another try.
4433        assert!(!round_is_clean(
4434            0,
4435            true,
4436            1,
4437            2,
4438            0,
4439            IncompleteReviewPolicy::Block
4440        ));
4441    }
4442
4443    #[test]
4444    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
4445        assert!(!round_is_clean(
4446            1,
4447            true,
4448            1,
4449            2,
4450            1,
4451            IncompleteReviewPolicy::Block
4452        ));
4453        assert!(!round_is_clean(
4454            0,
4455            false,
4456            1,
4457            2,
4458            1,
4459            IncompleteReviewPolicy::Block
4460        ));
4461    }
4462
4463    #[test]
4464    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
4465        // Every seat quota'd, nobody answered: there is no panel to decide
4466        // on, so this must fall through to the existing block-and-retry
4467        // fallback rather than call an unreviewed patch clean.
4468        assert!(!round_is_clean(
4469            0,
4470            true,
4471            0,
4472            2,
4473            2,
4474            IncompleteReviewPolicy::Block
4475        ));
4476    }
4477
4478    // `review_conclusion` is the exact decision the review hand-off task
4479    // fixed: a round budget spent (or a tree that stopped moving) must not
4480    // collapse into `Blocked` regardless of what verification actually
4481    // said. Deterministic and process-free for the same reason the
4482    // `round_is_clean` family above is.
4483    fn review_round(
4484        clean: bool,
4485        blocking: usize,
4486        answered: usize,
4487        expected: usize,
4488        progressed: bool,
4489        e2e_ok: bool,
4490    ) -> ReviewRound {
4491        ReviewRound {
4492            round: 1,
4493            head: "h".to_owned(),
4494            verified_head: None,
4495            reviews: Vec::new(),
4496            e2e: vec![CommandOutcome {
4497                command: "test".to_owned(),
4498                code: Some(if e2e_ok { 0 } else { 1 }),
4499                output_tail: String::new(),
4500                duration_ms: 0,
4501            }],
4502            verify_retried: false,
4503            e2e_deferred: false,
4504            e2e_defer_reason: None,
4505            fix: None,
4506            blocking,
4507            answered,
4508            expected,
4509            clean,
4510            progressed,
4511            vote_split: false,
4512            reconsideration: Vec::new(),
4513            verdict: None,
4514        }
4515    }
4516
4517    #[test]
4518    fn review_conclusion_is_none_when_nothing_has_run() {
4519        assert_eq!(review_conclusion(&[], 3), None);
4520    }
4521
4522    #[test]
4523    fn review_conclusion_is_none_while_rounds_remain() {
4524        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
4525        assert_eq!(review_conclusion(&rounds, 3), None);
4526    }
4527
4528    #[test]
4529    fn review_conclusion_is_gating_once_a_round_is_clean() {
4530        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
4531        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
4532    }
4533
4534    #[test]
4535    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
4536        let rounds = vec![
4537            review_round(false, 1, 2, 2, true, true),
4538            review_round(false, 1, 2, 2, true, true),
4539        ];
4540        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
4541    }
4542
4543    #[test]
4544    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
4545        let rounds = vec![
4546            review_round(false, 1, 2, 2, true, true),
4547            review_round(false, 1, 2, 2, true, false),
4548        ];
4549        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
4550    }
4551
4552    #[test]
4553    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
4554        // Missing input, not a verified tree — never a hand-off candidate.
4555        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
4556        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
4557    }
4558
4559    #[test]
4560    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
4561        let rounds = vec![
4562            review_round(false, 1, 2, 2, false, true),
4563            review_round(false, 1, 2, 2, false, true),
4564        ];
4565        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
4566    }
4567
4568    fn secs(n: u64) -> Duration {
4569        Duration::from_secs(n)
4570    }
4571
4572    /// A throwaway repo with one commit on `main`, for tests that need `merge`
4573    /// to make real (and, if it runs at all, real*ly fail*) git calls.
4574    fn init_repo(dir: &Path) {
4575        let run = |args: &[&str]| {
4576            let out = std::process::Command::new("git")
4577                .args(args)
4578                .current_dir(dir)
4579                .quiet()
4580                .output()
4581                .expect("spawn git");
4582            assert!(
4583                out.status.success(),
4584                "git {args:?} failed: {}",
4585                String::from_utf8_lossy(&out.stderr)
4586            );
4587        };
4588        run(&["init", "-b", "main"]);
4589        run(&["config", "user.name", "magi test"]);
4590        run(&["config", "user.email", "magi@example.com"]);
4591        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
4592        run(&["add", "-A"]);
4593        run(&["commit", "-m", "init"]);
4594    }
4595
4596    // `settle_questions` is what closes the ghost the phone showed: a run's
4597    // seat asked something, the run then ended, and nothing was left to
4598    // abandon the question it left `open`. `HOME` is a process-wide
4599    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
4600    // first time it runs in the binary — every test below still reaches the
4601    // same directory whichever call won, and each gets its own run id from
4602    // `RunState::new`, so they never collide there.
4603    fn ask_test_home() {
4604        crate::run::set_home(std::env::temp_dir().join("magi-graph-ask-tests-home"));
4605    }
4606
4607    /// A minimal, git-free `Runner` at a given status — `settle_questions`
4608    /// reads nothing else off it.
4609    fn runner_at(status: RunStatus) -> Runner {
4610        let mut state = RunState::new(
4611            PathBuf::from("/nonexistent/repo"),
4612            "main".to_owned(),
4613            "deadbeef".to_owned(),
4614            "task".to_owned(),
4615            Config::default(),
4616        );
4617        state.status = status;
4618        Runner {
4619            state,
4620            roles: ResolvedRoles {
4621                implementers: Vec::new(),
4622                judges: Vec::new(),
4623                reviewers: Vec::new(),
4624                fixer: None,
4625                conductor: conductor(),
4626            },
4627            sem: Arc::new(Semaphore::new(1)),
4628            pause: Pause::new(),
4629        }
4630    }
4631
4632    /// A fresh open question on `run`, stored and handed back for assertions.
4633    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
4634        let mut q = ask::Question::new(
4635            run.to_owned(),
4636            "implement".to_owned(),
4637            "impl-A".to_owned(),
4638            "Which storage backend should the cache use?".to_owned(),
4639            String::new(),
4640            vec!["SQLite".to_owned(), "Redis".to_owned()],
4641        );
4642        store.put(&mut q).unwrap();
4643        q
4644    }
4645
4646    #[test]
4647    fn a_failed_runs_open_question_is_abandoned() {
4648        ask_test_home();
4649        let store = ask::Questions::open();
4650        let mut runner = runner_at(RunStatus::Failed);
4651        let run = runner.state.id.clone();
4652        let q = ask_open_question(&store, &run);
4653
4654        runner.settle_questions();
4655
4656        let back = store.get(&q.id).unwrap();
4657        assert!(
4658            !back.status.open(),
4659            "the seat that asked died with the run; nobody is left to read an answer"
4660        );
4661        assert!(
4662            back.detail.contains(&run) && back.detail.contains("failed"),
4663            "the reason names what the run became, not just that it is gone: {}",
4664            back.detail
4665        );
4666    }
4667
4668    #[test]
4669    fn a_merged_runs_open_question_is_abandoned_too() {
4670        ask_test_home();
4671        let store = ask::Questions::open();
4672        // A run that finishes cleanly still leaves nobody to read an answer -
4673        // this is not only a failure-path cleanup.
4674        for status in [RunStatus::Merged, RunStatus::Ready] {
4675            let mut runner = runner_at(status);
4676            let run = runner.state.id.clone();
4677            let q = ask_open_question(&store, &run);
4678
4679            runner.settle_questions();
4680
4681            let back = store.get(&q.id).unwrap();
4682            assert!(
4683                !back.status.open(),
4684                "{status:?} run's question must not outlive the run"
4685            );
4686        }
4687    }
4688
4689    #[test]
4690    fn a_still_resumable_runs_open_question_is_left_alone() {
4691        ask_test_home();
4692        let store = ask::Questions::open();
4693        // `Blocked` and `Stalled` can still be resumed — the candidates, the
4694        // review round and the seat sessions are all still on disk — so a
4695        // question asked mid-round may yet get a real answer from a real
4696        // resume. Sweeping it here would be exactly the failure mode this
4697        // whole feature exists to avoid on the other side.
4698        for status in [RunStatus::Blocked, RunStatus::Stalled] {
4699            let mut runner = runner_at(status);
4700            let run = runner.state.id.clone();
4701            let q = ask_open_question(&store, &run);
4702
4703            runner.settle_questions();
4704
4705            let back = store.get(&q.id).unwrap();
4706            assert!(
4707                back.status.open(),
4708                "{status:?} is still alive; the question must still be waiting"
4709            );
4710        }
4711    }
4712
4713    #[test]
4714    fn settle_questions_never_touches_an_already_answered_question() {
4715        ask_test_home();
4716        let store = ask::Questions::open();
4717        let mut runner = runner_at(RunStatus::Failed);
4718        let run = runner.state.id.clone();
4719        let mut q = ask_open_question(&store, &run);
4720        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
4721            .unwrap();
4722        store.put(&mut q).unwrap();
4723
4724        // Called twice, the way a crash-recovered daemon reclaim and the
4725        // graph's own cleanup both can for the same run — `abandon_for_run`
4726        // only ever touches what is still open, so this must be inert both
4727        // times, not merely the second.
4728        runner.settle_questions();
4729        runner.settle_questions();
4730
4731        let back = store.get(&q.id).unwrap();
4732        assert_eq!(
4733            back.status,
4734            ask::QuestionStatus::Answered,
4735            "a real answer is a decision on record, never overwritten by a sweep"
4736        );
4737    }
4738
4739    /// `status == Ready` used to be read as "this is the harmless
4740    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
4741    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
4742    /// run's PR was closed without merging — and reentering `merge` with
4743    /// `mode` still `Pr` does not know the difference, so it pushed and
4744    /// opened a second pull request. `mode == Local` reproduces the same
4745    /// blind spot without a network call: reentry must not attempt another
4746    /// git merge once this node has already recorded an outcome.
4747    #[tokio::test]
4748    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
4749        let tmp = tempfile::tempdir().expect("tempdir");
4750        let repo = tmp.path().join("repo");
4751        std::fs::create_dir_all(&repo).unwrap();
4752        init_repo(&repo);
4753
4754        let mut config = Config::default();
4755        config.merge.mode = MergeMode::Local;
4756
4757        let mut state = RunState::new(
4758            repo.clone(),
4759            "main".to_owned(),
4760            "deadbeef".to_owned(),
4761            "task".to_owned(),
4762            config,
4763        );
4764        state.candidates = vec![Candidate {
4765            index: 0,
4766            label: 'A',
4767            agent: "alpha".to_owned(),
4768            branch: "does-not-exist".to_owned(),
4769            worktree: repo.clone(),
4770            summary: String::new(),
4771            stat: String::new(),
4772            files: 0,
4773            commits: 0,
4774            empty: false,
4775            failed: None,
4776            duration_ms: 0,
4777            folded: false,
4778        }];
4779        state.tally = Some(Tally {
4780            first_choice: BTreeMap::from([('A', 1)]),
4781            borda: BTreeMap::new(),
4782            winner: 'A',
4783            rankings: 1,
4784            unanimous_initial: true,
4785            deliberated: false,
4786            changed_votes: 0,
4787            unanimous_final: true,
4788            tie_break: None,
4789            judges: 0,
4790            present: 0,
4791            quorum: 0,
4792            met_quorum: true,
4793            uncontested: Some("only candidate A produced a change".to_owned()),
4794        });
4795        state.reviews = vec![ReviewRound {
4796            round: 1,
4797            head: "deadbeef".to_owned(),
4798            verified_head: None,
4799            reviews: Vec::new(),
4800            e2e: Vec::new(),
4801            fix: None,
4802            blocking: 0,
4803            answered: 0,
4804            expected: 0,
4805            clean: true,
4806            verify_retried: false,
4807            e2e_deferred: false,
4808            e2e_defer_reason: None,
4809            progressed: false,
4810            vote_split: false,
4811            reconsideration: Vec::new(),
4812            verdict: None,
4813        }];
4814        state.gate = vec![CommandOutcome {
4815            command: "test".to_owned(),
4816            code: Some(0),
4817            output_tail: String::new(),
4818            duration_ms: 0,
4819        }];
4820        // Reached its conclusion already — e.g. `land` closing the PR without
4821        // merging it, which (like the honest `MergeMode::None` path) leaves
4822        // `status` at `Ready`. The recorded outcome is what actually marks
4823        // this node done.
4824        state.status = RunStatus::Ready;
4825        state.merge = Some(MergeOutcome {
4826            mode: MergeMode::Local,
4827            ok: false,
4828            detail: "already concluded".to_owned(),
4829        });
4830
4831        let mut runner = Runner {
4832            state,
4833            roles: ResolvedRoles {
4834                implementers: Vec::new(),
4835                judges: Vec::new(),
4836                reviewers: Vec::new(),
4837                fixer: None,
4838                conductor: conductor(),
4839            },
4840            sem: Arc::new(Semaphore::new(1)),
4841            pause: Pause::new(),
4842        };
4843
4844        runner.merge().await.expect("merge");
4845
4846        assert_eq!(
4847            runner.state.status,
4848            RunStatus::Ready,
4849            "a concluded run's status must not change on reentry"
4850        );
4851        assert_eq!(
4852            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
4853            Some("already concluded"),
4854            "merge must not run again once the node already recorded an outcome"
4855        );
4856    }
4857
4858    #[tokio::test]
4859    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
4860        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
4861        let tmp = tempfile::tempdir().expect("tempdir");
4862        let repo = tmp.path().join("repo");
4863        std::fs::create_dir_all(&repo).unwrap();
4864        init_repo(&repo);
4865
4866        let mut config = Config::default();
4867        config.merge.mode = MergeMode::Pr;
4868        config.graph.land = true;
4869        config.graph.land_approval = false;
4870
4871        let mut state = RunState::new(
4872            repo.clone(),
4873            "main".to_owned(),
4874            "deadbeef".to_owned(),
4875            "task".to_owned(),
4876            config,
4877        );
4878        state.candidates = vec![Candidate {
4879            index: 0,
4880            label: 'A',
4881            agent: "alpha".to_owned(),
4882            branch: "does-not-exist".to_owned(),
4883            worktree: repo.clone(),
4884            summary: String::new(),
4885            stat: String::new(),
4886            files: 0,
4887            commits: 0,
4888            empty: false,
4889            failed: None,
4890            duration_ms: 0,
4891            folded: false,
4892        }];
4893        state.tally = Some(Tally {
4894            first_choice: BTreeMap::from([('A', 1)]),
4895            borda: BTreeMap::new(),
4896            winner: 'A',
4897            rankings: 1,
4898            unanimous_initial: true,
4899            deliberated: false,
4900            changed_votes: 0,
4901            unanimous_final: true,
4902            tie_break: None,
4903            judges: 0,
4904            present: 0,
4905            quorum: 0,
4906            met_quorum: true,
4907            uncontested: Some("only candidate A produced a change".to_owned()),
4908        });
4909        state.reviews = vec![ReviewRound {
4910            round: 1,
4911            head: "deadbeef".to_owned(),
4912            verified_head: None,
4913            reviews: Vec::new(),
4914            e2e: Vec::new(),
4915            fix: None,
4916            blocking: 0,
4917            answered: 0,
4918            expected: 0,
4919            clean: true,
4920            verify_retried: false,
4921            e2e_deferred: false,
4922            e2e_defer_reason: None,
4923            progressed: false,
4924            vote_split: false,
4925            reconsideration: Vec::new(),
4926            verdict: None,
4927        }];
4928        state.gate = vec![CommandOutcome {
4929            command: "test".to_owned(),
4930            code: Some(0),
4931            output_tail: String::new(),
4932            duration_ms: 0,
4933        }];
4934        // A first pass through `merge` already pushed and opened this pull
4935        // request; `status` is `Landing` because a previous call into `land`
4936        // parked or was interrupted before it reached a terminal outcome.
4937        state.status = RunStatus::Landing;
4938        state.merge = Some(MergeOutcome {
4939            mode: MergeMode::Pr,
4940            ok: true,
4941            detail: "https://example.invalid/x/y/pull/1".to_owned(),
4942        });
4943
4944        // The Landing-resume shortcut calls `run_land` directly rather than
4945        // through `merge`, which is exactly the call site that used to skip
4946        // `settle_questions` - see the fixture below.
4947        ask_test_home();
4948        let store = ask::Questions::open();
4949        let q = ask_open_question(&store, &state.id);
4950
4951        let mut runner = Runner {
4952            state,
4953            roles: ResolvedRoles {
4954                implementers: Vec::new(),
4955                judges: Vec::new(),
4956                reviewers: Vec::new(),
4957                fixer: None,
4958                conductor: conductor(),
4959            },
4960            sem: Arc::new(Semaphore::new(1)),
4961            pause: Pause::new(),
4962        };
4963
4964        // `execute`, not `merge` directly: the Landing-resume shortcut lives
4965        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
4966        // exactly because `review_loop` would otherwise clobber the marker
4967        // first.
4968        runner.execute().await.expect("execute");
4969
4970        assert_eq!(
4971            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
4972            Some("https://example.invalid/x/y/pull/1"),
4973            "reentry must not push again or open a second pull request over the \
4974             one `land` is already watching"
4975        );
4976        assert_ne!(
4977            runner.state.status,
4978            RunStatus::Landing,
4979            "land could not actually reach the fake pull request, so it must \
4980             have given up rather than left the run silently parked forever"
4981        );
4982        // `land` could not reach the fake pull request, so it gave up into
4983        // `Blocked` - still resumable, so the question must not have been
4984        // swept just because this branch now also calls `settle_questions`.
4985        assert_eq!(runner.state.status, RunStatus::Blocked);
4986        assert!(
4987            store.get(&q.id).unwrap().status.open(),
4988            "Blocked is still alive; settle_questions must have been a no-op here"
4989        );
4990    }
4991
4992    fn state_with_round(round: ReviewRound) -> RunState {
4993        let mut s = RunState::new(
4994            PathBuf::from("/repo"),
4995            "main".to_owned(),
4996            "abc1234".to_owned(),
4997            "add retries".to_owned(),
4998            Config::default(),
4999        );
5000        s.reviews = vec![round];
5001        s
5002    }
5003
5004    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
5005        crate::verdict::Finding {
5006            id: id.to_owned(),
5007            severity,
5008            file: None,
5009            line: None,
5010            title: title.to_owned(),
5011            detail: String::new(),
5012        }
5013    }
5014
5015    #[test]
5016    fn pr_body_names_open_findings_and_declined_ones() {
5017        let round = ReviewRound {
5018            round: 2,
5019            head: "deadbee".to_owned(),
5020            verified_head: None,
5021            reviews: vec![ReviewRecord {
5022                reviewer: 1,
5023                agent: "alpha".to_owned(),
5024                summary: String::new(),
5025                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
5026                vote: None,
5027                failed: None,
5028                duration_ms: 0,
5029            }],
5030            e2e: vec![CommandOutcome {
5031                command: "cargo test".to_owned(),
5032                code: Some(0),
5033                output_tail: String::new(),
5034                duration_ms: 0,
5035            }],
5036            verify_retried: false,
5037            e2e_deferred: false,
5038            e2e_defer_reason: None,
5039            fix: Some(FixRecord {
5040                agent: "alpha".to_owned(),
5041                addressed: Vec::new(),
5042                rejected: vec![crate::verdict::Rejection {
5043                    id: "R1-1-1".to_owned(),
5044                    why: "not reachable from any caller".to_owned(),
5045                }],
5046                notes: String::new(),
5047                committed: true,
5048                failed: None,
5049                duration_ms: 0,
5050            }),
5051            blocking: 0,
5052            answered: 1,
5053            expected: 1,
5054            clean: false,
5055            progressed: true,
5056            vote_split: false,
5057            reconsideration: Vec::new(),
5058            verdict: None,
5059        };
5060        let state = state_with_round(round);
5061        let body = pr_body(&state, 'A');
5062
5063        assert!(body.contains("add retries"), "the task must still be there");
5064        assert!(body.contains("R2-1-1"), "{body}");
5065        assert!(body.contains("unused import"), "{body}");
5066        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
5067        assert!(
5068            body.contains("not reachable from any caller"),
5069            "the reason it was declined: {body}"
5070        );
5071    }
5072
5073    #[test]
5074    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
5075        let round = ReviewRound {
5076            round: 1,
5077            head: "deadbee".to_owned(),
5078            verified_head: None,
5079            reviews: vec![ReviewRecord {
5080                reviewer: 1,
5081                agent: "alpha".to_owned(),
5082                summary: String::new(),
5083                findings: Vec::new(),
5084                vote: None,
5085                failed: None,
5086                duration_ms: 0,
5087            }],
5088            e2e: Vec::new(),
5089            verify_retried: false,
5090            e2e_deferred: false,
5091            e2e_defer_reason: None,
5092            fix: None,
5093            blocking: 0,
5094            answered: 1,
5095            expected: 1,
5096            clean: true,
5097            progressed: false,
5098            vote_split: false,
5099            reconsideration: Vec::new(),
5100            verdict: None,
5101        };
5102        let state = state_with_round(round);
5103        let body = pr_body(&state, 'A');
5104        assert!(!body.contains("Open review findings"), "{body}");
5105        assert!(!body.contains("Declined"), "{body}");
5106    }
5107
5108    #[test]
5109    fn pr_body_titles_itself_from_the_task_not_run_or_candidate() {
5110        let state = RunState::new(
5111            PathBuf::from("/repo"),
5112            "main".to_owned(),
5113            "abc1234".to_owned(),
5114            "add retries".to_owned(),
5115            Config::default(),
5116        );
5117        let body = pr_body(&state, 'A');
5118        let title = body.lines().next().unwrap();
5119
5120        assert_eq!(
5121            title, "add retries",
5122            "the title must be the task, not run/candidate bookkeeping: {body}"
5123        );
5124        assert!(
5125            body.contains(&format!("magi:run/{}", state.id)),
5126            "the run id must still be recoverable from the footer: {body}"
5127        );
5128        assert!(
5129            body.contains("magi:candidate-a"),
5130            "the candidate must still be recoverable from the footer: {body}"
5131        );
5132    }
5133
5134    #[test]
5135    fn pr_body_never_titles_itself_off_a_blank_first_line() {
5136        let leading_blank = RunState::new(
5137            PathBuf::from("/repo"),
5138            "main".to_owned(),
5139            "abc1234".to_owned(),
5140            "\n\n  \nadd retries\n\ndetails".to_owned(),
5141            Config::default(),
5142        );
5143        let body = pr_body(&leading_blank, 'A');
5144        assert_eq!(
5145            body.lines().next(),
5146            Some("add retries"),
5147            "a leading blank line must not become an empty title: {body}"
5148        );
5149
5150        let whitespace_only = RunState::new(
5151            PathBuf::from("/repo"),
5152            "main".to_owned(),
5153            "abc1234".to_owned(),
5154            "   \n  \n".to_owned(),
5155            Config::default(),
5156        );
5157        let body = pr_body(&whitespace_only, 'A');
5158        let title = body.lines().next().unwrap_or_default();
5159        assert!(
5160            !title.is_empty(),
5161            "a whitespace-only instruction must still fall back to a non-empty title: {body}"
5162        );
5163    }
5164
5165    #[test]
5166    fn manual_merge_command_matches_the_configured_style() {
5167        let repo = Path::new("/repo");
5168        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
5169
5170        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
5171        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
5172
5173        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
5174        assert_eq!(
5175            squash,
5176            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
5177             \"Merge magi run 0832 (candidate A)\""
5178        );
5179
5180        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
5181        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
5182    }
5183
5184    #[test]
5185    fn a_nudge_gets_a_quarter_of_the_budget() {
5186        // The judge and implement budgets magi ships with.
5187        assert_eq!(retry_budget(secs(1200), true), secs(300));
5188        assert_eq!(retry_budget(secs(3600), true), secs(900));
5189    }
5190
5191    #[test]
5192    fn a_resent_prompt_keeps_the_whole_budget() {
5193        // The seat kept no context, so the retry is the original job again and
5194        // shortening it would only guarantee a second failure.
5195        assert_eq!(retry_budget(secs(1200), false), secs(1200));
5196        assert_eq!(retry_budget(secs(60), false), secs(60));
5197    }
5198
5199    #[test]
5200    fn the_floor_never_exceeds_the_original_budget() {
5201        // A short configured timeout must not be *raised* by the floor: the
5202        // operator asked for a bound, and a retry may not outlast the attempt
5203        // it is retrying.
5204        assert_eq!(retry_budget(secs(60), true), secs(60));
5205        assert_eq!(retry_budget(secs(480), true), secs(120));
5206        assert_eq!(retry_budget(secs(0), true), secs(0));
5207    }
5208}