Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::agent::{self, AgentOutput, Invocation, SeatState};
30use crate::blind;
31use crate::bump;
32use crate::config::{
33    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, Prompts, ResolvedRoles,
34};
35use crate::git;
36use crate::land;
37use crate::proc::Quiet as _;
38use crate::prompt::{
39    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
40};
41use crate::run::{
42    BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
43    MergeOutcome, QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus,
44    Tally, VoteRecord, tail, write_artifact,
45};
46use crate::verdict::{
47    self, FinalVote, FixReport, Position, Ranking, Review, ReviewRevote, ReviewVote, Severity,
48};
49
50/// How much verification output is kept and fed back to the fixer.
51const OUTPUT_TAIL: usize = 8_000;
52
53/// Bytes of a failing command's output kept in an event, so the reason a run
54/// stopped is readable from the report without opening `run.json`.
55const EVENT_OUTPUT_TAIL: usize = 2_000;
56
57/// Consecutive review rounds with no tree progress (see
58/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
59/// instead of spending the rest of the round budget.
60///
61/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
62/// legitimately finds nothing left to change (its previous round's fix already
63/// covered it, and this round's reviewers re-raised only nits) looks the same
64/// as one that is spinning, for exactly one round. Two in a row is where the
65/// two stop being distinguishable, and a review round on this workload has
66/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
67/// third attempt at a tree that has not moved twice running is pure cost.
68/// This does not touch `review_rounds` itself, which stays the operator's
69/// call.
70pub(crate) const STAGNANT_LIMIT: usize = 2;
71
72/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
73/// a base that moved before giving up and leaving the run `Blocked` for a
74/// person.
75///
76/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
77/// that keeps moving faster than a run can catch it is not something more
78/// rebasing fixes, it is a person's call. Not the same *number as*
79/// `land_rounds` - this budget is spent before a pull request exists, land's
80/// after - but bounded for the identical reason, so it uses the same
81/// default. Counted across both call sites in [`Runner::finish_after_tally`]
82/// (once before review, once before the gate), because either one finding
83/// the base still moving is the same signal.
84const BASE_SYNC_ROUNDS: usize = 4;
85
86/// One queued agent invocation.
87///
88/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
89/// CLI hung up on its own stream is asked again from the same job rather than
90/// rebuilt from scratch. See [`Runner::resume_undelivered`].
91#[derive(Clone)]
92struct SeatJob {
93    spec: AgentSpec,
94    seat: SeatState,
95    cwd: PathBuf,
96    prompt: String,
97    timeout: Duration,
98    allow_write: bool,
99    sessions: bool,
100    artifacts: PathBuf,
101    stem: String,
102}
103
104/// How the graph reads one agent invocation.
105///
106/// Quota is split out from an ordinary failure on purpose: a rate-limited call
107/// is known to fail again if retried now, so the retry loop must not spend an
108/// attempt on it. `Dropped` is split out for the opposite reason: unlike
109/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
110/// error JSON, never the agent's answer — a caller that matched only
111/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
112/// left to read that JSON as if it were usable output. `resume_undelivered`
113/// is the only caller that acts on it; everywhere else it is reported like an
114/// ordinary failure.
115enum AgentOutcome {
116    /// A usable output.
117    Ok(AgentOutput),
118    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
119    Quota(AgentOutput),
120    /// The CLI hung up on its own stream after billed work. See
121    /// [`agent::AgentOutput::work_undelivered`].
122    Dropped(AgentOutput),
123    /// Any other failure: a timeout, a bad exit code, an empty reply.
124    Failed(String),
125}
126
127/// A request to park the run at its next node boundary.
128///
129/// Cloning is how the request travels: the loop keeps one handle and hands a
130/// clone to each [`Runner`], and every clone points at the same flag. There
131/// is no channel because there is nothing to send - the only message is
132/// "park", it is idempotent, and a flag cannot be missed by a receiver that
133/// was not listening yet.
134///
135/// The boundary is what makes this cheap. Every node writes the run's state
136/// before the next one starts, and every node skips what is already recorded:
137/// `prep` returns early once candidates exist, `implement` asks only the seats
138/// with nothing on disk, `judge` returns early once judgements exist. So a
139/// parked run resumes into exactly the node it stopped before, and no agent
140/// work is thrown away. Killing the process mid-node, by contrast, loses
141/// whatever the seats in flight had not yet written - which for an implement
142/// wave is an hour of paid work.
143#[derive(Debug, Clone, Default)]
144pub struct Pause(Arc<AtomicBool>);
145
146impl Pause {
147    /// A pause nobody has asked for yet.
148    #[must_use]
149    pub fn new() -> Self {
150        Self::default()
151    }
152
153    /// Ask the run to park at its next node boundary. Idempotent.
154    pub fn park(&self) {
155        self.0.store(true, Ordering::SeqCst);
156    }
157
158    /// Has a park been asked for?
159    #[must_use]
160    pub fn parked(&self) -> bool {
161        self.0.load(Ordering::SeqCst)
162    }
163}
164
165/// Drives one run.
166pub struct Runner {
167    /// Run state; public so the CLI can report on it.
168    pub state: RunState,
169    roles: ResolvedRoles,
170    sem: Arc<Semaphore>,
171    /// Set when someone wants the run parked at its next node boundary.
172    pause: Pause,
173}
174
175/// The commit a run branches from: the base branch as the remote has it.
176///
177/// Two failures this replaces. A run used to branch off `HEAD` and so refused
178/// to start on a dirty tree, which made `magi serve` decline every task for as
179/// long as the operator had work in progress - most of the time. Branching off
180/// the *local* base branch fixed that and introduced a worse one: `land` merges
181/// the winner on GitHub, nothing updates the local ref, and the next run
182/// branches off a base missing everything the previous runs landed. Two tasks
183/// in a row from a phone would have had the second silently re-implementing
184/// against stale code and opening a pull request that reverted the first.
185///
186/// Only refs move here - no checkout, no local branch, no merge - so it is safe
187/// with uncommitted work in the tree. A machine with no network still starts:
188/// the fetch may fail and the local tip is used with a warning, because
189/// refusing to run offline is a worse failure than running against a base the
190/// operator can see for themselves.
191///
192/// One function, called by both entry points. Two answers to "where does a run
193/// branch from" is the kind of drift nobody notices until a diff is wrong.
194async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
195    let tracking = format!("{remote}/{base_branch}");
196    let fetched = git::fetch(repo, remote, base_branch).await;
197    if let Ok(out) = &fetched
198        && out.ok()
199        && git::rev_exists(repo, &tracking).await
200    {
201        return git::rev_parse(repo, &tracking).await;
202    }
203    let why = match &fetched {
204        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
205        Ok(_) => format!("{remote} has no {base_branch}"),
206        Err(e) => e.to_string(),
207    };
208    tracing::warn!(
209        "could not read {tracking} ({why}); branching off the local \
210         {base_branch} instead, which may be behind"
211    );
212    git::rev_parse(repo, base_branch).await.with_context(|| {
213        format!(
214            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
215             branch that exists"
216        )
217    })
218}
219
220impl Runner {
221    /// Start a fresh run against `repo`.
222    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
223        let repo = git::toplevel(repo).await?;
224        let missing = agent::missing_programs(&config.agents);
225        if !missing.is_empty() {
226            bail!(
227                "these agent programs are not on PATH: {}. Fix the roster in \
228                 magi.toml or install them.",
229                missing.join(", ")
230            );
231        }
232        let base_branch = match config.merge.base.clone() {
233            Some(b) => b,
234            None => git::current_branch(&repo)
235                .await?
236                .context("HEAD is detached; set [merge] base in magi.toml")?,
237        };
238        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
239        // Still worth saying out loud. The operator's uncommitted work is not
240        // part of this run, and someone watching a candidate fail to use a
241        // change they just made deserves to know why.
242        if !git::is_clean(&repo).await? {
243            tracing::warn!(
244                "{} has uncommitted changes; they are not part of this run, \
245                 which branches off {base_branch} ({})",
246                repo.display(),
247                &base_commit[..base_commit.len().min(8)]
248            );
249        }
250        let roles = config.resolve_roles()?;
251        let max_parallel = config.graph.max_parallel.max(1);
252        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
253        state.event("start", format!("run {} created", state.id));
254        state.save()?;
255        Ok(Self {
256            state,
257            roles,
258            sem: Arc::new(Semaphore::new(max_parallel)),
259            pause: Pause::new(),
260        })
261    }
262
263    /// Open a review-only run against work that already exists on `branch`.
264    ///
265    /// The expensive half of the graph is the implement wave — measured at
266    /// 111 and 134 internal tool-loop turns on this repository, against a
267    /// handful for a judge or a reviewer. The cheap half is worth running on
268    /// hand-written work too, and there was no way to reach it.
269    ///
270    /// No new state and no schema change are needed: a run with **one** viable
271    /// candidate and a tally already decided degrades `execute` to exactly
272    /// review → gate → merge, because `judge` skips a single-candidate field,
273    /// `deliberate` has fewer than two first choices to reconcile, `vote`
274    /// returns early, `tally` is already present and `fold_losers` has no
275    /// losers. Resuming such a run therefore does the right thing as well.
276    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
277        let repo = git::toplevel(repo).await?;
278        let missing = agent::missing_programs(&config.agents);
279        if !missing.is_empty() {
280            bail!(
281                "these agent programs are not on PATH: {}. Fix the roster in \
282                 magi.toml or install them.",
283                missing.join(", ")
284            );
285        }
286        if !git::branch_exists(&repo, branch).await? {
287            bail!("no branch `{branch}` in {}", repo.display());
288        }
289        let base_branch = match config.merge.base.clone() {
290            Some(b) => b,
291            None => git::current_branch(&repo)
292                .await?
293                .context("HEAD is detached; set [merge] base in magi.toml")?,
294        };
295        if base_branch == branch {
296            bail!("`{branch}` is the base branch; there is nothing to review against");
297        }
298        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
299
300        let roles = config.resolve_roles()?;
301        let max_parallel = config.graph.max_parallel.max(1);
302        // The commit subjects are the closest thing to a task statement that
303        // existing work carries, and the reviewers are told as much.
304        let log = git::log_oneline(&repo, &base_commit, branch)
305            .await
306            .unwrap_or_default();
307        let instruction = format!(
308            "Review the work already on branch `{branch}`. There is no task \
309             statement: what the change claims to do is whatever its commits \
310             say.\n\n{}",
311            if log.trim().is_empty() {
312                "(no commit messages)"
313            } else {
314                log.trim()
315            }
316        );
317        let mut state = RunState::new(
318            repo.clone(),
319            base_branch,
320            base_commit.clone(),
321            instruction,
322            config,
323        );
324
325        // An attached worktree, so the fixer's commits land on the branch under
326        // review rather than on a detached head nobody will look at again.
327        let worktree = state.worktree_root().join("under-review");
328        if let Some(parent) = worktree.parent() {
329            tokio::fs::create_dir_all(parent).await.ok();
330        }
331        let path = worktree.to_string_lossy().to_string();
332        git::git(&repo, &["worktree", "add", &path, branch])
333            .await
334            .with_context(|| {
335                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
336            })?;
337
338        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
339            .await
340            .unwrap_or(0);
341        if commits == 0 {
342            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
343        }
344        let files = git::changed_files(&worktree, &base_commit, "HEAD")
345            .await
346            .map(|f| f.len())
347            .unwrap_or(0);
348        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
349            .await
350            .unwrap_or_default();
351
352        state.candidates.push(Candidate {
353            index: 0,
354            label: 'A',
355            // Not an agent id on purpose: nothing in the roster wrote this, and
356            // the stats tables must not credit anyone with a win for it.
357            agent: "(existing branch)".to_owned(),
358            branch: branch.to_owned(),
359            worktree,
360            summary: String::new(),
361            stat,
362            files,
363            commits,
364            empty: false,
365            failed: None,
366            duration_ms: 0,
367            folded: false,
368        });
369        state.tally = Some(Tally {
370            first_choice: BTreeMap::from([('A', 0)]),
371            borda: BTreeMap::new(),
372            winner: 'A',
373            rankings: 0,
374            unanimous_initial: false,
375            deliberated: false,
376            changed_votes: 0,
377            unanimous_final: false,
378            tie_break: None,
379            // No panel sat, so no quorum applies. Zero judges is the correct
380            // number for work that never competed, and must not be reported as
381            // a collapsed panel.
382            judges: 0,
383            present: 0,
384            quorum: 0,
385            met_quorum: true,
386            uncontested: Some("review-only run: nothing competed".to_owned()),
387        });
388        state.status = RunStatus::Reviewing;
389        state.event(
390            "start",
391            format!(
392                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
393                state.id
394            ),
395        );
396        state.save()?;
397        Ok(Self {
398            state,
399            roles,
400            sem: Arc::new(Semaphore::new(max_parallel)),
401            pause: Pause::new(),
402        })
403    }
404
405    /// Reopen an existing run.
406    pub fn resume(id: &str) -> Result<Self> {
407        let state = RunState::load(id)?;
408        let roles = state.config.resolve_roles()?;
409        let max_parallel = state.config.graph.max_parallel.max(1);
410        Ok(Self {
411            state,
412            roles,
413            sem: Arc::new(Semaphore::new(max_parallel)),
414            pause: Pause::new(),
415        })
416    }
417
418    /// Walk the graph to a terminal state, skipping nodes already recorded.
419    pub async fn execute(&mut self) -> Result<()> {
420        // Moving again, so it is no longer parked. Set before the walk rather
421        // than in `resume`, so every way of re-entering the graph clears it
422        // and a card cannot claim a run is waiting to be resumed while the
423        // agents are already working.
424        self.state.parked = false;
425        // Any seat this state still lists as answering belongs to whatever
426        // process last drove this run — this one included, if it crashed
427        // mid-wave. Cleared and flushed immediately, before anything else
428        // runs, so a resume can never show a seat as live when nothing is
429        // asking it anything yet; the node that actually dispatches the next
430        // wave repopulates it.
431        if self.state.clear_active() {
432            self.state.save()?;
433        }
434        // A run that already lost its quorum never resumes into the verdict
435        // machinery: `deliberate` and `vote` would otherwise clobber the
436        // stalled marker back to Voting and the run would keep going past a
437        // verdict that is no longer trustworthy. Everything already recorded is
438        // kept, so the run stays resumable (or foldable) for a human to pick up.
439        //
440        // On --resume the run gets one chance to repair itself: the seats a
441        // rate limit took out are re-asked. If their quota has since reset and
442        // the quorum is restored, the run picks up and finishes; otherwise it
443        // stays stale and still-resumable for a later retry. If it does not
444        // recover, the returned status stays `Stalled` and nothing was
445        // clobbered (the recovery only mutates entries for the lost seats).
446        if self.state.status == RunStatus::Stalled {
447            if self.recover_stall().await? {
448                self.finish_after_tally().await?;
449            } else {
450                // Still below quorum: persist the marker and stay resumable.
451                self.state.save()?;
452            }
453            return Ok(());
454        }
455        // A run parked inside `land` - watching CI, mid fix-round, or
456        // waiting on the owner's merge approval - resumes directly into it,
457        // never back through `prep`. Everything before `merge` already
458        // concluded; that is the only way `status` reaches `Landing` in the
459        // first place. Re-walking `review_loop` first would also be actively
460        // wrong: its own status recomputation (see its doc) treats any
461        // clean round as reason to set `status` to `Gating`, which would
462        // clobber this marker before `merge` ever ran, and this run would
463        // never find its way back into `land` at all.
464        if self.state.status == RunStatus::Landing {
465            self.run_land().await?;
466            return Ok(());
467        }
468        self.prep().await?;
469        if self.park_here()? {
470            return Ok(());
471        }
472        self.implement().await?;
473        if self.park_here()? {
474            return Ok(());
475        }
476        self.judge().await?;
477        if self.park_here()? {
478            return Ok(());
479        }
480        self.deliberate().await?;
481        if self.park_here()? {
482            return Ok(());
483        }
484        self.vote().await?;
485        if self.park_here()? {
486            return Ok(());
487        }
488        self.tally()?;
489        // A verdict that lost its quorum is not trustworthy: do not review,
490        // gate, or merge on it. Everything already done is kept, so the run
491        // stays resumable (or foldable); the human can replace the agent that
492        // ran out of quota and pick it up.
493        if self.state.status == RunStatus::Stalled {
494            // Persist the stalled marker now — the normal end-of-execute save
495            // below is below this early return, and without it a resumed run
496            // would reload a pre-tally status and keep going.
497            self.state.save()?;
498            return Ok(());
499        }
500        self.finish_after_tally().await?;
501        Ok(())
502    }
503
504    /// Park here if asked to, recording it in the run's own timeline.
505    ///
506    /// Returns whether the caller should stop walking the graph. The state is
507    /// saved either way by the node that just finished; this adds the event so
508    /// the operator's card says why a run that is neither finished nor moving
509    /// is sitting where it is.
510    fn park_here(&mut self) -> Result<bool> {
511        if !self.pause.parked() {
512            return Ok(false);
513        }
514        self.state.event(
515            "park",
516            format!(
517                "parked after `{}` — resume to carry on from here",
518                self.state.status.as_str()
519            ),
520        );
521        self.state.parked = true;
522        self.state.save()?;
523        Ok(true)
524    }
525
526    /// Hand the runner a pause to watch.
527    pub fn on_pause(&mut self, pause: Pause) {
528        self.pause = pause;
529    }
530
531    /// The tail of the graph after a trustworthy tally: fold losers, review,
532    /// gate, merge, and persist.
533    async fn finish_after_tally(&mut self) -> Result<()> {
534        self.fold_losers().await?;
535        // Before review starts, and again right before the gate: a run's
536        // review rounds can themselves take long enough for the base to move
537        // a second time, and the gate is the one node whose "green" gets
538        // acted on.
539        self.sync_to_base().await?;
540        self.review_loop().await?;
541        self.sync_to_base().await?;
542        self.gate().await?;
543        self.merge().await?;
544        self.state.save()?;
545        Ok(())
546    }
547
548    // ---------------------------------------------------------------- prep
549
550    async fn prep(&mut self) -> Result<()> {
551        if !self.state.candidates.is_empty() {
552            return Ok(());
553        }
554        self.state.status = RunStatus::Prep;
555        let repo = self.state.repo.clone();
556        let base = self.state.base_commit.clone();
557        let root = self.state.worktree_root();
558        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
559
560        // The hook is the write-time half of the blindness contract; the
561        // presentation filter in `blind` is the half that cannot be bypassed.
562        let hooks_dir = self.state.dir().join("hooks");
563        if self.state.config.blind.commit_msg_hook {
564            std::fs::create_dir_all(&hooks_dir)
565                .with_context(|| format!("create {}", hooks_dir.display()))?;
566            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
567            let path = hooks_dir.join("commit-msg");
568            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
569            make_executable(&path)?;
570            // Ref-counted rather than a plain idempotent set: with more than
571            // one run able to be in flight in the same repository at once
572            // (see `Config::daemon.max_concurrent_runs`), a bare "already
573            // true?" check cannot tell "another run of mine still needs
574            // this" from "nobody does", and the run that happens to finish
575            // first would disable the hook out from under a sibling still
576            // relying on it.
577            git::acquire_worktree_config(&repo).await?;
578            self.state.enabled_worktree_config = true;
579        }
580
581        for (index, (spec, label)) in self
582            .roles
583            .implementers
584            .clone()
585            .into_iter()
586            .zip(labels)
587            .enumerate()
588        {
589            let branch = self.state.branch_for(label);
590            let worktree = root.join(format!("cand-{label}"));
591            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
592            if self.state.config.blind.commit_msg_hook {
593                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
594            }
595            git::local_exclude(&worktree, "/.magi/").await?;
596            self.state.candidates.push(Candidate {
597                index,
598                label,
599                agent: spec.id.clone(),
600                branch,
601                worktree,
602                summary: String::new(),
603                stat: String::new(),
604                files: 0,
605                commits: 0,
606                empty: false,
607                failed: None,
608                duration_ms: 0,
609                folded: false,
610            });
611        }
612
613        for j in 1..=self.roles.judges.len() {
614            let wt = root.join(format!("judge-{j}"));
615            if !wt.exists() {
616                git::worktree_add_detached(&repo, &wt, &base).await?;
617            }
618        }
619
620        // A judge cannot tell it is looking at its own patch — the seats keep
621        // separate conversations — but a panel that shares agents with the
622        // field is less independent than it looks, and that is worth saying out
623        // loud once per run rather than leaving it in the config.
624        let authors: Vec<&str> = self
625            .roles
626            .implementers
627            .iter()
628            .map(|a| a.id.as_str())
629            .collect();
630        let overlap: Vec<String> = self
631            .roles
632            .judges
633            .iter()
634            .enumerate()
635            .filter(|(_, j)| authors.contains(&j.id.as_str()))
636            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
637            .collect();
638        if !overlap.is_empty() {
639            let note = format!(
640                "{} also authored a candidate; blind, but the panel is less \
641                 independent than {} distinct agents would be",
642                overlap.join(", "),
643                self.roles.judges.len()
644            );
645            self.state.event("prep", note);
646        }
647
648        self.state.event(
649            "prep",
650            format!(
651                "{} candidates, {} judges, base {} ({})",
652                self.state.candidates.len(),
653                self.roles.judges.len(),
654                &self.state.base_commit[..7.min(self.state.base_commit.len())],
655                self.state.base_branch
656            ),
657        );
658        self.state.status = RunStatus::Implementing;
659        self.state.save()?;
660        Ok(())
661    }
662
663    // ----------------------------------------------------------- implement
664
665    async fn implement(&mut self) -> Result<()> {
666        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
667        // agent files with `magi task add` name the run that paid for it. The
668        // prompt overlay is cloned alongside it because the waves borrow it
669        // while `self` is mutably borrowed by the node's own bookkeeping.
670        let run_id = self.state.id.clone();
671        let prompts = self.state.config.prompts.clone();
672        let todo: Vec<usize> = self
673            .state
674            .candidates
675            .iter()
676            .enumerate()
677            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
678            .map(|(i, _)| i)
679            .collect();
680        if todo.is_empty() {
681            return self.after_implement();
682        }
683        self.state.status = RunStatus::Implementing;
684
685        let language = self.state.config.graph.language.clone();
686        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
687        let sessions = self.state.config.graph.sessions;
688        let artifacts = agent::artifacts_dir(&self.state.dir());
689
690        let mut jobs = Vec::new();
691        for &i in &todo {
692            let (index, label, worktree) = {
693                let c = &self.state.candidates[i];
694                (c.index, c.label, c.worktree.clone())
695            };
696            let spec = self.roles.implementers[index].clone();
697            let seat_key = format!("impl-{label}");
698            let seat = self.seat(&seat_key, &spec.id);
699            let instruction = self.state.instruction.clone();
700            jobs.push(SeatJob {
701                spec,
702                seat,
703                prompt: prompt::implement(&instruction, &worktree.to_string_lossy(), &language),
704                cwd: worktree,
705                timeout,
706                allow_write: true,
707                sessions,
708                artifacts: artifacts.clone(),
709                stem: format!("impl-{label}"),
710            });
711        }
712
713        self.state.event(
714            "implement",
715            format!("{} candidates in parallel", jobs.len()),
716        );
717        // Kept so a seat whose CLI hung up can be asked again from the same
718        // job: `wave` consumes what it is given.
719        let sent = jobs.clone();
720        let cache = self.state.config.cache_dir();
721        let ctx = WaveCtx {
722            run: &run_id,
723            node: "implement",
724            prompts: &prompts,
725            cache: cache.as_deref(),
726        };
727        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
728        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
729            .await;
730
731        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
732            let seat_key = seat.key.clone();
733            self.state.seats.insert(seat.key.clone(), seat);
734            let label = self.state.candidates[i].label;
735            let worktree = self.state.candidates[i].worktree.clone();
736            let base = self.state.base_commit.clone();
737
738            let (summary, duration, failed) = match out {
739                AgentOutcome::Ok(o) => {
740                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
741                    let failed = (!o.usable()).then(|| {
742                        if o.timed_out {
743                            "agent timed out".to_owned()
744                        } else {
745                            format!("agent exited with {:?}", o.exit_code)
746                        }
747                    });
748                    (text, o.duration_ms, failed)
749                }
750                // Left un-resumed by `resume_undelivered` (a dirty tree
751                // already rescues the work, or there was no session left to
752                // resume into) — reported like the ordinary failure it is,
753                // never as if `o.text` (the CLI's raw error JSON) were an
754                // answer.
755                AgentOutcome::Dropped(o) => {
756                    let why = o
757                        .dropped
758                        .as_ref()
759                        .map(|d| d.why.as_str())
760                        .unwrap_or("the CLI ended the stream without delivering its answer");
761                    (
762                        String::new(),
763                        o.duration_ms,
764                        Some(format!("the CLI dropped the stream ({why})")),
765                    )
766                }
767                AgentOutcome::Quota(o) => {
768                    self.state.quota.push(QuotaLoss {
769                        seat: seat_key,
770                        node: "implement".to_owned(),
771                        at: Timestamp::now(),
772                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
773                    });
774                    (
775                        String::new(),
776                        o.duration_ms,
777                        Some("rate limited (quota); produced no change".to_owned()),
778                    )
779                }
780                AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
781            };
782
783            // Rescue anything the agent edited but never committed: an
784            // uncommitted candidate would silently be an empty one.
785            let rescued = git::commit_all(
786                &worktree,
787                &format!("magi: candidate {label} (uncommitted work)"),
788            )
789            .await
790            .unwrap_or(false);
791            let commits = git::commits_ahead(&worktree, &base, "HEAD")
792                .await
793                .unwrap_or(0);
794            let patch = git::diff(&worktree, &base, "HEAD")
795                .await
796                .unwrap_or_default();
797            let stat = git::diff_stat(&worktree, &base, "HEAD")
798                .await
799                .unwrap_or_default();
800            let files = git::changed_files(&worktree, &base, "HEAD")
801                .await
802                .map(|f| f.len())
803                .unwrap_or(0);
804            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
805
806            let c = &mut self.state.candidates[i];
807            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
808            c.stat = stat;
809            c.files = files;
810            c.commits = commits;
811            c.duration_ms = duration;
812            c.empty = commits == 0 || patch.trim().is_empty();
813            // An agent that failed but still produced a committed change stays
814            // in the running: the patch is what gets judged, not the exit code.
815            c.failed = match failed {
816                Some(_) if c.empty => failed,
817                _ => None,
818            };
819            let note = match (&c.failed, c.empty, rescued) {
820                (Some(e), _, _) => format!("candidate {label}: {e}"),
821                (None, true, _) => format!("candidate {label}: no change produced"),
822                (None, false, true) => {
823                    format!(
824                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
825                    )
826                }
827                (None, false, false) => {
828                    format!("candidate {label}: {files} files, {commits} commits")
829                }
830            };
831            self.state.event("implement", note);
832            self.state.save()?;
833        }
834
835        self.after_implement()
836    }
837
838    /// Ask again, once, for work a CLI did and then failed to hand over.
839    ///
840    /// [`agent::dropped_stream`] recognises the one shape observed: an error
841    /// status with an empty response and a usage report showing output tokens,
842    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
843    /// seven minutes and 14,267 output tokens that arrived as an empty
844    /// candidate, because `agy`'s own subscriber fell behind and hung up.
845    ///
846    /// Two conditions, and both matter:
847    ///
848    /// - **Only when the tree is untouched.** Often the agent has already
849    ///   written its files and only the closing message was lost; the rescue
850    ///   commit below picks that up and there is nothing to ask for. Re-asking
851    ///   then would pay for a second implementation of work already on disk.
852    /// - **Once.** A CLI that drops one stream can drop the next, and this
853    ///   node is the most expensive in the graph.
854    ///
855    /// The re-ask is a resume, not a re-run: `has_context` is true because the
856    /// dropped reply still carried its `conversation_id`, so the seat is asked
857    /// to finish what it was doing rather than sent the whole task again. It
858    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
859    /// node's - for the same reason a re-ranked judge does: restating finished
860    /// work is not the work.
861    ///
862    /// Unlike a quota this is worth retrying at all: a rate limit fails the
863    /// same way until it resets, while an abandoned conversation is still
864    /// there to be picked up.
865    async fn resume_undelivered(
866        &mut self,
867        results: &mut [(usize, SeatState, AgentOutcome)],
868        sent: &[SeatJob],
869        prompts: &Prompts,
870        run_id: &str,
871    ) {
872        for (wi, seat, out) in results.iter_mut() {
873            let Some(dropped) = (match &*out {
874                AgentOutcome::Dropped(o) => o.dropped.clone(),
875                _ => None,
876            }) else {
877                continue;
878            };
879            let Some(job) = sent.get(*wi) else { continue };
880            // Already on disk? Then only the closing message was lost.
881            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
882                self.state.event(
883                    "implement",
884                    format!(
885                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
886                         work is in the tree",
887                        seat.key, dropped.output_tokens, dropped.why
888                    ),
889                );
890                continue;
891            }
892            // The re-ask only makes sense as a resume: `resume_after_drop`
893            // says nothing about the task, trusting the seat to still hold it.
894            // Without a session to resume — sessions disabled, or this CLI's
895            // drop shape happened not to carry a session id — that prompt
896            // would open a brand-new conversation with no context at all,
897            // which is worse than leaving this as the ordinary failure it
898            // already is.
899            if !has_context(&job.spec, seat, job.sessions) {
900                self.state.event(
901                    "implement",
902                    format!(
903                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
904                         is no session left to resume",
905                        seat.key, dropped.output_tokens, dropped.why
906                    ),
907                );
908                continue;
909            }
910            self.state.event(
911                "implement",
912                format!(
913                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
914                     conversation",
915                    seat.key, dropped.output_tokens, dropped.why
916                ),
917            );
918            let mut retry = job.clone();
919            retry.seat = seat.clone();
920            retry.prompt = prompt::resume_after_drop(&dropped.why);
921            retry.timeout = retry_budget(job.timeout, true);
922            retry.stem = format!("{}-resume", job.stem);
923            let cache = self.state.config.cache_dir();
924            let ctx = WaveCtx {
925                run: run_id,
926                node: "implement",
927                prompts,
928                cache: cache.as_deref(),
929            };
930            let (resumed_seat, resumed) =
931                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
932            *seat = resumed_seat;
933            *out = resumed;
934        }
935    }
936
937    fn after_implement(&mut self) -> Result<()> {
938        // Scan every candidate patch once the set is complete.
939        if self.state.leaks.is_empty() {
940            let cfg = self.state.config.blind.clone();
941            let mut leaks = Vec::new();
942            for c in &self.state.candidates {
943                let Some(patch) =
944                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
945                else {
946                    continue;
947                };
948                leaks.extend(blind::scan(
949                    &format!("candidate {} patch", c.label),
950                    &patch,
951                    &cfg.vendor_tokens,
952                ));
953            }
954            if !leaks.is_empty() {
955                let summary = leaks
956                    .iter()
957                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
958                    .collect::<Vec<_>>()
959                    .join(", ");
960                match cfg.on_leak {
961                    LeakPolicy::Fail => {
962                        self.state.status = RunStatus::Failed;
963                        self.state
964                            .event("blind", format!("vendor text in a patch: {summary}"));
965                        self.state.leaks = leaks;
966                        self.state.save()?;
967                        bail!(
968                            "blind.on_leak = \"fail\" and vendor text reached a \
969                             judged patch: {summary}"
970                        );
971                    }
972                    LeakPolicy::Redact => self.state.event(
973                        "blind",
974                        format!("redacting vendor text for judging: {summary}"),
975                    ),
976                    LeakPolicy::Warn => self.state.event(
977                        "blind",
978                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
979                    ),
980                }
981                self.state.leaks = leaks;
982            }
983        }
984
985        if self.state.viable().is_empty() {
986            self.state.status = RunStatus::Failed;
987            self.state.save()?;
988            bail!("no candidate produced a change; nothing to judge");
989        }
990        self.state.status = RunStatus::Judging;
991        self.state.save()?;
992        Ok(())
993    }
994
995    // --------------------------------------------------------------- judge
996
997    async fn judge(&mut self) -> Result<()> {
998        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
999        // agent files with `magi task add` name the run that paid for it. The
1000        // prompt overlay is cloned alongside it because the waves borrow it
1001        // while `self` is mutably borrowed by the node's own bookkeeping.
1002        let run_id = self.state.id.clone();
1003        let prompts = self.state.config.prompts.clone();
1004        if !self.state.judgements.is_empty() || self.state.judge_skipped {
1005            return Ok(());
1006        }
1007        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1008        if viable.len() == 1 {
1009            // Recorded so this is a one-time event: `judgements` stays empty
1010            // either way, which without this flag is indistinguishable from
1011            // "not yet judged" on the next reentry — and status is left
1012            // untouched, so a later node's conclusion (e.g. `Blocked` after
1013            // the review budget ran out) survives a resume instead of being
1014            // clobbered back to `Judging` by this node running again.
1015            self.state.judge_skipped = true;
1016            self.state.event(
1017                "judge",
1018                format!(
1019                    "only candidate {} produced a change; judging skipped",
1020                    viable[0].label
1021                ),
1022            );
1023            self.state.save()?;
1024            return Ok(());
1025        }
1026        self.state.status = RunStatus::Judging;
1027
1028        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1029        let language = self.state.config.graph.language.clone();
1030        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1031        let sessions = self.state.config.graph.sessions;
1032        let artifacts = agent::artifacts_dir(&self.state.dir());
1033        let root = self.state.worktree_root();
1034        let base_short = short(&self.state.base_commit);
1035
1036        let mut jobs = Vec::new();
1037        let mut orders = Vec::new();
1038        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1039            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1040            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1041            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
1042            let seat_key = format!("judge-{}", j + 1);
1043            let seat = self.seat(&seat_key, &spec.id);
1044            jobs.push(SeatJob {
1045                prompt: prompt::judge(
1046                    &self.state.instruction,
1047                    &views,
1048                    self.roles.judges.len(),
1049                    &base_short,
1050                    &language,
1051                ),
1052                spec,
1053                seat,
1054                cwd: root.join(format!("judge-{}", j + 1)),
1055                timeout,
1056                allow_write: false,
1057                sessions,
1058                artifacts: artifacts.clone(),
1059                stem: format!("judge-{}", j + 1),
1060            });
1061        }
1062
1063        self.state.event(
1064            "judge",
1065            format!(
1066                "{} judges ranking {} candidates blind",
1067                jobs.len(),
1068                viable.len()
1069            ),
1070        );
1071        let labels_for_check = labels.clone();
1072        let mut quota_losses = Vec::new();
1073        let cache = self.state.config.cache_dir();
1074        let ctx = WaveCtx {
1075            run: &run_id,
1076            node: "judge",
1077            prompts: &prompts,
1078            cache: cache.as_deref(),
1079        };
1080        let results = ask_json_wave::<Ranking>(
1081            jobs,
1082            Arc::clone(&self.sem),
1083            self.state.config.graph.retries,
1084            &ctx,
1085            &mut quota_losses,
1086            &mut self.state,
1087            &move |r: &Ranking| r.validate(&labels_for_check),
1088        )
1089        .await;
1090        self.state.quota.extend(quota_losses);
1091
1092        for (j, (seat, res)) in results.into_iter().enumerate() {
1093            let agent_id = seat.agent.clone();
1094            self.state.seats.insert(seat.key.clone(), seat);
1095            let mut record = Judgement {
1096                judge: j + 1,
1097                seat: format!("judge-{}", j + 1),
1098                agent: agent_id,
1099                ranking: Vec::new(),
1100                reasons: BTreeMap::new(),
1101                confidence: None,
1102                order: orders[j].clone(),
1103                failed: None,
1104                duration_ms: 0,
1105            };
1106            match res {
1107                Ok((ranking, out)) => {
1108                    record.ranking = ranking.normalized();
1109                    record.reasons = ranking.reasons;
1110                    record.confidence = ranking.confidence;
1111                    record.duration_ms = out.duration_ms;
1112                    self.state.event(
1113                        "judge",
1114                        format!(
1115                            "judge {} ranked {}",
1116                            j + 1,
1117                            record.ranking.iter().collect::<String>()
1118                        ),
1119                    );
1120                }
1121                Err(e) => {
1122                    record.failed = Some(e.to_string());
1123                    self.state
1124                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1125                }
1126            }
1127            self.state.judgements.push(record);
1128            self.state.save()?;
1129        }
1130        Ok(())
1131    }
1132
1133    // ---------------------------------------------------------- deliberate
1134
1135    async fn deliberate(&mut self) -> Result<()> {
1136        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1137        // agent files with `magi task add` name the run that paid for it. The
1138        // prompt overlay is cloned alongside it because the waves borrow it
1139        // while `self` is mutably borrowed by the node's own bookkeeping.
1140        let run_id = self.state.id.clone();
1141        let prompts = self.state.config.prompts.clone();
1142        if !self.state.deliberation.is_empty() {
1143            return Ok(());
1144        }
1145        let tops: Vec<char> = self
1146            .state
1147            .judgements
1148            .iter()
1149            .filter_map(|j| j.ranking.first().copied())
1150            .collect();
1151        let rounds = self.state.config.graph.deliberate_rounds;
1152        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1153            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1154                self.state.event(
1155                    "deliberate",
1156                    format!("judges agreed on {} outright; no deliberation", tops[0]),
1157                );
1158            }
1159            self.state.status = RunStatus::Voting;
1160            self.state.save()?;
1161            return Ok(());
1162        }
1163
1164        self.state.status = RunStatus::Deliberating;
1165        self.state.event(
1166            "deliberate",
1167            format!(
1168                "split: first choices were {} — opening {rounds} round(s)",
1169                tops.iter().collect::<String>()
1170            ),
1171        );
1172
1173        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1174        let language = self.state.config.graph.language.clone();
1175        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1176        let sessions = self.state.config.graph.sessions;
1177        let artifacts = agent::artifacts_dir(&self.state.dir());
1178        let root = self.state.worktree_root();
1179        let base_short = short(&self.state.base_commit);
1180
1181        // Judges argue in sequence so that a turn can answer the one before it;
1182        // that is the difference between deliberation and three parallel
1183        // monologues.
1184        for round in 1..=rounds {
1185            let mut turns: Vec<DeliberationTurn> = Vec::new();
1186            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1187                if self.state.judgements[j].failed.is_some() {
1188                    continue;
1189                }
1190                let seat_key = format!("judge-{}", j + 1);
1191                let mut seat = self.seat(&seat_key, &spec.id);
1192                let transcript = self.transcript(&turns, j);
1193                let context = if has_context(&spec, &seat, sessions) {
1194                    None
1195                } else {
1196                    Some(self.candidate_block(&viable, &base_short))
1197                };
1198                let text = prompt::deliberate(
1199                    &self.state.instruction,
1200                    context.as_deref(),
1201                    &transcript,
1202                    round,
1203                    rounds,
1204                    &language,
1205                );
1206                let job = SeatJob {
1207                    spec,
1208                    seat: seat.clone(),
1209                    prompt: text,
1210                    cwd: root.join(format!("judge-{}", j + 1)),
1211                    timeout,
1212                    allow_write: false,
1213                    sessions,
1214                    artifacts: artifacts.clone(),
1215                    stem: format!("delib-{round}-judge-{}", j + 1),
1216                };
1217                let cache = self.state.config.cache_dir();
1218                let ctx = WaveCtx {
1219                    run: &run_id,
1220                    node: "deliberate",
1221                    prompts: &prompts,
1222                    cache: cache.as_deref(),
1223                };
1224                let (updated, out) =
1225                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1226                seat = updated;
1227                let agent_id = seat.agent.clone();
1228                let seat_key = seat.key.clone();
1229                self.state.seats.insert(seat.key.clone(), seat);
1230                let body = match out {
1231                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1232                    // Never read the CLI's raw error JSON as this judge's
1233                    // position — skip the seat instead, the same as any other
1234                    // failed turn.
1235                    AgentOutcome::Dropped(o) => {
1236                        let why =
1237                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1238                                "the CLI ended the stream without delivering its answer",
1239                            );
1240                        self.state.event(
1241                            "deliberate",
1242                            format!(
1243                                "judge {} skipped: the CLI dropped the stream ({why})",
1244                                j + 1
1245                            ),
1246                        );
1247                        continue;
1248                    }
1249                    AgentOutcome::Quota(o) => {
1250                        self.state.quota.push(QuotaLoss {
1251                            seat: seat_key,
1252                            node: "deliberate".to_owned(),
1253                            at: Timestamp::now(),
1254                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1255                        });
1256                        self.state.event(
1257                            "deliberate",
1258                            format!("judge {} skipped: rate limited (quota)", j + 1),
1259                        );
1260                        continue;
1261                    }
1262                    AgentOutcome::Failed(e) => {
1263                        self.state
1264                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1265                        continue;
1266                    }
1267                };
1268                let tentative = verdict::extract_json::<Position>(&body)
1269                    .ok()
1270                    .and_then(|p| p.tentative)
1271                    .and_then(|s| s.trim().chars().next())
1272                    .map(|c| c.to_ascii_uppercase());
1273                self.state.event(
1274                    "deliberate",
1275                    format!(
1276                        "round {round}: judge {} now favours {}",
1277                        j + 1,
1278                        tentative.map_or("—".to_owned(), |c| c.to_string())
1279                    ),
1280                );
1281                turns.push(DeliberationTurn {
1282                    judge: j + 1,
1283                    agent: agent_id,
1284                    body: blind::sanitize_prose(&body, &self.state.config.blind),
1285                    tentative,
1286                });
1287            }
1288            self.state
1289                .deliberation
1290                .push(DeliberationRound { round, turns });
1291            self.state.save()?;
1292        }
1293
1294        self.state.status = RunStatus::Voting;
1295        self.state.save()?;
1296        Ok(())
1297    }
1298
1299    // ---------------------------------------------------------------- vote
1300
1301    async fn vote(&mut self) -> Result<()> {
1302        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1303        // agent files with `magi task add` name the run that paid for it. The
1304        // prompt overlay is cloned alongside it because the waves borrow it
1305        // while `self` is mutably borrowed by the node's own bookkeeping.
1306        let run_id = self.state.id.clone();
1307        let prompts = self.state.config.prompts.clone();
1308        if !self.state.votes.is_empty() {
1309            return Ok(());
1310        }
1311        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1312        if viable.len() == 1 {
1313            return Ok(());
1314        }
1315        self.state.status = RunStatus::Voting;
1316
1317        let language = self.state.config.graph.language.clone();
1318        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1319        let sessions = self.state.config.graph.sessions;
1320        let artifacts = agent::artifacts_dir(&self.state.dir());
1321        let root = self.state.worktree_root();
1322        let base_short = short(&self.state.base_commit);
1323        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1324
1325        let mut jobs = Vec::new();
1326        let mut seats_at = Vec::new();
1327        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1328            if self
1329                .state
1330                .judgements
1331                .get(j)
1332                .is_some_and(|r| r.failed.is_some())
1333            {
1334                continue;
1335            }
1336            let seat_key = format!("judge-{}", j + 1);
1337            let seat = self.seat(&seat_key, &spec.id);
1338            let mut text = prompt::final_vote(&viable, &language);
1339            if !has_context(&spec, &seat, sessions) {
1340                text = format!(
1341                    "{}\n\n# Candidates\n\n{}",
1342                    text,
1343                    self.candidate_block(&candidates, &base_short)
1344                );
1345            }
1346            jobs.push(SeatJob {
1347                spec,
1348                seat,
1349                prompt: text,
1350                cwd: root.join(format!("judge-{}", j + 1)),
1351                timeout,
1352                allow_write: false,
1353                sessions,
1354                artifacts: artifacts.clone(),
1355                stem: format!("vote-judge-{}", j + 1),
1356            });
1357            seats_at.push(j);
1358        }
1359
1360        self.state.event(
1361            "vote",
1362            format!(
1363                "collecting {} final votes one by one, privately",
1364                jobs.len()
1365            ),
1366        );
1367        let allowed = viable.clone();
1368        let mut quota_losses = Vec::new();
1369        let cache = self.state.config.cache_dir();
1370        let ctx = WaveCtx {
1371            run: &run_id,
1372            node: "vote",
1373            prompts: &prompts,
1374            cache: cache.as_deref(),
1375        };
1376        let results = ask_json_wave::<FinalVote>(
1377            jobs,
1378            Arc::clone(&self.sem),
1379            self.state.config.graph.retries,
1380            &ctx,
1381            &mut quota_losses,
1382            &mut self.state,
1383            &move |v: &FinalVote| match v.label() {
1384                Some(c) if allowed.contains(&c) => Ok(()),
1385                other => bail!("vote {other:?} is not one of {allowed:?}"),
1386            },
1387        )
1388        .await;
1389        self.state.quota.extend(quota_losses);
1390
1391        for (&j, (seat, res)) in seats_at.iter().zip(results) {
1392            let agent_id = seat.agent.clone();
1393            self.state.seats.insert(seat.key.clone(), seat);
1394            let initial = self
1395                .state
1396                .judgements
1397                .get(j)
1398                .and_then(|r| r.ranking.first().copied());
1399            let mut record = VoteRecord {
1400                judge: j + 1,
1401                agent: agent_id,
1402                vote: None,
1403                reason: String::new(),
1404                changed: false,
1405            };
1406            match res {
1407                Ok((v, _)) => {
1408                    record.vote = v.label();
1409                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1410                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1411                    self.state.event(
1412                        "vote",
1413                        format!(
1414                            "judge {} voted {}{}",
1415                            j + 1,
1416                            record.vote.unwrap_or('?'),
1417                            if record.changed { " (changed)" } else { "" }
1418                        ),
1419                    );
1420                }
1421                Err(e) => {
1422                    self.state
1423                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1424                }
1425            }
1426            self.state.votes.push(record);
1427            self.state.save()?;
1428        }
1429        Ok(())
1430    }
1431
1432    // --------------------------------------------------------------- tally
1433
1434    fn tally(&mut self) -> Result<()> {
1435        if self.state.tally.is_some() {
1436            return Ok(());
1437        }
1438        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1439        let tops: Vec<char> = self
1440            .state
1441            .judgements
1442            .iter()
1443            .filter_map(|j| j.ranking.first().copied())
1444            .collect();
1445        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1446
1447        // A judge whose private vote failed still counted once, in the initial
1448        // ranking; using it beats discarding a whole seat.
1449        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1450        let mut cast: Vec<char> = Vec::new();
1451        for (i, j) in self.state.judgements.iter().enumerate() {
1452            let vote = self
1453                .state
1454                .votes
1455                .iter()
1456                .find(|v| v.judge == i + 1)
1457                .and_then(|v| v.vote)
1458                .or_else(|| j.ranking.first().copied());
1459            if let Some(v) = vote {
1460                *first_choice.entry(v).or_insert(0) += 1;
1461                cast.push(v);
1462            }
1463        }
1464
1465        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1466        for j in &self.state.judgements {
1467            let n = j.ranking.len();
1468            for (pos, label) in j.ranking.iter().enumerate() {
1469                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1470            }
1471        }
1472
1473        let best = first_choice.values().copied().max().unwrap_or(0);
1474        let mut leaders: Vec<char> = first_choice
1475            .iter()
1476            .filter(|(_, v)| **v == best)
1477            .map(|(k, _)| *k)
1478            .collect();
1479        let mut tie_break = None;
1480        if leaders.len() > 1 {
1481            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1482            let borda_leaders: Vec<char> = leaders
1483                .iter()
1484                .copied()
1485                .filter(|l| borda[l] == top_borda)
1486                .collect();
1487            tie_break = Some(if borda_leaders.len() == 1 {
1488                format!(
1489                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1490                    leaders.len()
1491                )
1492            } else {
1493                format!(
1494                    "{} way tie on both first-choice votes and Borda points, broken by label order",
1495                    leaders.len()
1496                )
1497            });
1498            leaders = borda_leaders;
1499            leaders.sort_unstable();
1500        }
1501        let winner = *leaders
1502            .first()
1503            .or(viable.first())
1504            .context("no candidate to declare a winner from")?;
1505
1506        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1507        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1508        let deliberated = !self.state.deliberation.is_empty();
1509
1510        // Whose verdict is this? A rate-limited seat is absent even if it
1511        // ranked before the limit hit, so presence is measured against the
1512        // recorded losses, not just "did a ranking ever appear".
1513        let quota_seats: std::collections::BTreeSet<&str> =
1514            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1515        let mut present = 0usize;
1516        for (i, j) in self.state.judgements.iter().enumerate() {
1517            if quota_seats.contains(j.seat.as_str()) {
1518                continue;
1519            }
1520            let ranked = !j.ranking.is_empty() && j.failed.is_none();
1521            let voted = self
1522                .state
1523                .votes
1524                .iter()
1525                .any(|v| v.judge == i + 1 && v.vote.is_some());
1526            if ranked || voted {
1527                present += 1;
1528            }
1529        }
1530        // Strict majority of the configured panel. A bare majority is real
1531        // signal we can act on, while a minority verdict must never stand in
1532        // for a healthy one. A one-candidate run needs no panel at all, and
1533        // `judges` stays `0` rather than the roster size a panel that never
1534        // sat would otherwise be credited with.
1535        let needs_quorum = viable.len() > 1;
1536        let judges_total = if needs_quorum {
1537            self.roles.judges.len()
1538        } else {
1539            0
1540        };
1541        let quorum = if needs_quorum {
1542            judges_total / 2 + 1
1543        } else {
1544            0
1545        };
1546        let met_quorum = !needs_quorum || present >= quorum;
1547        let uncontested = (!needs_quorum).then(|| {
1548            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1549        });
1550
1551        self.state.event(
1552            "tally",
1553            match &uncontested {
1554                Some(reason) => format!("winner {winner} — {reason}"),
1555                None => format!(
1556                    "winner {winner} — votes {} | initial {} | {} changed | \
1557                     {present}/{judges_total} judges{}",
1558                    first_choice
1559                        .iter()
1560                        .map(|(k, v)| format!("{k}:{v}"))
1561                        .collect::<Vec<_>>()
1562                        .join(" "),
1563                    if unanimous_initial {
1564                        "unanimous"
1565                    } else {
1566                        "split"
1567                    },
1568                    changed_votes,
1569                    if met_quorum {
1570                        String::new()
1571                    } else {
1572                        format!(" — below quorum ({quorum} required)")
1573                    },
1574                ),
1575            },
1576        );
1577        if !met_quorum {
1578            self.state.event(
1579                "stall",
1580                format!(
1581                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1582                     the run stops here, resumable"
1583                ),
1584            );
1585        }
1586        self.state.tally = Some(Tally {
1587            first_choice,
1588            borda,
1589            winner,
1590            rankings: tops.len(),
1591            unanimous_initial,
1592            deliberated,
1593            changed_votes,
1594            unanimous_final,
1595            tie_break,
1596            judges: judges_total,
1597            present,
1598            quorum,
1599            met_quorum,
1600            uncontested,
1601        });
1602        self.state.status = if met_quorum {
1603            RunStatus::Reviewing
1604        } else {
1605            RunStatus::Stalled
1606        };
1607        self.state.save()?;
1608        Ok(())
1609    }
1610
1611    // ------------------------------------------------------------- recover
1612
1613    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
1614    /// resumed toward completion once the transient cause clears.
1615    ///
1616    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
1617    /// it toward the quorum, which is exactly the set of seats whose absence
1618    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
1619    /// not depend on which node happened to hit the limit), or an ordinary
1620    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
1621    /// seat is never disturbed.
1622    ///
1623    /// A seat that now answers with a usable ranking is "recovered": its
1624    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
1625    /// `tally` counts it present again), and its vote re-collected. A seat that
1626    /// still fails keeps its loss and stays absent.
1627    ///
1628    /// Returns `true` when the re-tally restores the quorum (the run may proceed
1629    /// to review/gate/merge), `false` when it is still below quorum (the run
1630    /// stays `Stalled`, still resumable for a later retry).
1631    #[allow(clippy::too_many_lines)]
1632    async fn recover_stall(&mut self) -> Result<bool> {
1633        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1634        // agent files with `magi task add` name the run that paid for it. The
1635        // prompt overlay is cloned alongside it because the waves borrow it
1636        // while `self` is mutably borrowed by the node's own bookkeeping.
1637        let run_id = self.state.id.clone();
1638        let prompts = self.state.config.prompts.clone();
1639        // Absent seats = quota-lost at any node, or failed outright. Mirroring
1640        // `tally`'s presence test (rather than the old quota-judge/vote filter)
1641        // is what keeps a non-quota collapse — or a quota loss recorded at the
1642        // deliberate node — from being a permanent dead-end on `--resume`.
1643        let quota_seats: BTreeSet<&str> =
1644            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1645        let absent: Vec<String> = self
1646            .state
1647            .judgements
1648            .iter()
1649            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
1650            .map(|j| j.seat.clone())
1651            .collect();
1652        if absent.is_empty() {
1653            return Ok(false);
1654        }
1655        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1656        if viable.len() <= 1 {
1657            return Ok(false);
1658        }
1659        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1660        let language = self.state.config.graph.language.clone();
1661        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1662        let sessions = self.state.config.graph.sessions;
1663        let artifacts = agent::artifacts_dir(&self.state.dir());
1664        let root = self.state.worktree_root();
1665        let base_short = short(&self.state.base_commit);
1666        let candidates: Vec<Candidate> = viable.clone();
1667
1668        // Map each absent seat key to its 0-based position in `roles.judges`.
1669        let mut positions: Vec<usize> = absent
1670            .iter()
1671            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
1672            .collect();
1673        if positions.is_empty() {
1674            return Ok(false);
1675        }
1676        positions.sort_unstable();
1677        positions.dedup();
1678
1679        // Re-rank the lost seats, one blind prompt each.
1680        let mut judge_jobs = Vec::new();
1681        for &j in &positions {
1682            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1683            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1684            let seat_key = format!("judge-{}", j + 1);
1685            let spec = self.roles.judges[j].clone();
1686            let seat = self.seat(&seat_key, &spec.id);
1687            judge_jobs.push(SeatJob {
1688                spec,
1689                seat,
1690                prompt: prompt::judge(
1691                    &self.state.instruction,
1692                    &views,
1693                    self.roles.judges.len(),
1694                    &base_short,
1695                    &language,
1696                ),
1697                cwd: root.join(seat_key),
1698                timeout,
1699                allow_write: false,
1700                sessions,
1701                artifacts: artifacts.clone(),
1702                stem: format!("judge-{}-recover", j + 1),
1703            });
1704        }
1705
1706        let labels_for_check = labels.clone();
1707        let mut judge_losses = Vec::new();
1708        let retries = self.state.config.graph.retries;
1709        let cache = self.state.config.cache_dir();
1710        let ctx = WaveCtx {
1711            run: &run_id,
1712            node: "judge",
1713            prompts: &prompts,
1714            cache: cache.as_deref(),
1715        };
1716        let results = ask_json_wave::<Ranking>(
1717            judge_jobs,
1718            Arc::clone(&self.sem),
1719            retries,
1720            &ctx,
1721            &mut judge_losses,
1722            &mut self.state,
1723            &move |r: &Ranking| r.validate(&labels_for_check),
1724        )
1725        .await;
1726
1727        // Refresh the judgement of every seat that ranked again.
1728        let mut recovered: BTreeSet<usize> = BTreeSet::new();
1729        for (&j, (seat, res)) in positions.iter().zip(results) {
1730            self.state.seats.insert(seat.key.clone(), seat);
1731            let record = &mut self.state.judgements[j];
1732            match res {
1733                Ok((ranking, out)) => {
1734                    record.ranking = ranking.normalized();
1735                    record.reasons = ranking.reasons;
1736                    record.confidence = ranking.confidence;
1737                    record.failed = None;
1738                    record.duration_ms = out.duration_ms;
1739                    recovered.insert(j);
1740                    self.state.event(
1741                        "recover",
1742                        format!("judge {} ranked again after the limit", j + 1),
1743                    );
1744                }
1745                Err(e) => {
1746                    self.state
1747                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
1748                }
1749            }
1750        }
1751
1752        // Re-ask the votes of the seats that recovered a ranking.
1753        let mut vote_jobs = Vec::new();
1754        let mut vote_pos: Vec<usize> = Vec::new();
1755        for &j in &recovered {
1756            let seat_key = format!("judge-{}", j + 1);
1757            let spec = self.roles.judges[j].clone();
1758            let seat = self.seat(&seat_key, &spec.id);
1759            let mut text = prompt::final_vote(&labels, &language);
1760            if !has_context(&spec, &seat, sessions) {
1761                text = format!(
1762                    "{}\n\n# Candidates\n\n{}",
1763                    text,
1764                    self.candidate_block(&candidates, &base_short)
1765                );
1766            }
1767            vote_jobs.push(SeatJob {
1768                spec,
1769                seat,
1770                prompt: text,
1771                cwd: root.join(seat_key),
1772                timeout,
1773                allow_write: false,
1774                sessions,
1775                artifacts: artifacts.clone(),
1776                stem: format!("vote-judge-{}-recover", j + 1),
1777            });
1778            vote_pos.push(j);
1779        }
1780        let allowed = labels.clone();
1781        let mut vote_losses = Vec::new();
1782        let vote_retries = self.state.config.graph.retries;
1783        let vote_cache = self.state.config.cache_dir();
1784        let ctx = WaveCtx {
1785            run: &run_id,
1786            node: "vote",
1787            prompts: &prompts,
1788            cache: vote_cache.as_deref(),
1789        };
1790        let votes = ask_json_wave::<FinalVote>(
1791            vote_jobs,
1792            Arc::clone(&self.sem),
1793            vote_retries,
1794            &ctx,
1795            &mut vote_losses,
1796            &mut self.state,
1797            &move |v: &FinalVote| match v.label() {
1798                Some(c) if allowed.contains(&c) => Ok(()),
1799                other => bail!("vote {other:?} is not one of {allowed:?}"),
1800            },
1801        )
1802        .await;
1803        for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
1804            let agent_id = seat.agent.clone();
1805            self.state.seats.insert(seat.key.clone(), seat);
1806            match res {
1807                Ok((v, _)) => {
1808                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
1809                        rec.vote = v.label();
1810                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1811                    } else {
1812                        self.state.votes.push(VoteRecord {
1813                            judge: j + 1,
1814                            agent: agent_id,
1815                            vote: v.label(),
1816                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
1817                            changed: false,
1818                        });
1819                    }
1820                    self.state.event(
1821                        "recover",
1822                        format!("judge {} voted again after the limit", j + 1),
1823                    );
1824                }
1825                Err(e) => {
1826                    self.state
1827                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
1828                }
1829            }
1830        }
1831
1832        // A seat that ranked again is present even if its re-vote failed —
1833        // `tally` falls back to the initial ranking's first choice — so clear
1834        // its quota loss. Seats that still fail keep theirs and stay absent.
1835        if !recovered.is_empty() {
1836            let recovered_keys: BTreeSet<String> = recovered
1837                .iter()
1838                .map(|&j| format!("judge-{}", j + 1))
1839                .collect();
1840            self.state
1841                .quota
1842                .retain(|q| !recovered_keys.contains(&q.seat));
1843        }
1844
1845        // Recompute the verdict from the refreshed panel.
1846        self.state.tally = None;
1847        self.tally()?;
1848        Ok(self
1849            .state
1850            .tally
1851            .as_ref()
1852            .map(|t| t.met_quorum)
1853            .unwrap_or(false))
1854    }
1855
1856    // ----------------------------------------------------------------- fold
1857
1858    async fn fold_losers(&mut self) -> Result<()> {
1859        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
1860            return Ok(());
1861        };
1862        let repo = self.state.repo.clone();
1863        let mut folded = Vec::new();
1864        for i in 0..self.state.candidates.len() {
1865            let c = &self.state.candidates[i];
1866            if c.label == winner || c.folded {
1867                continue;
1868            }
1869            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
1870            git::worktree_remove(&repo, &wt).await.ok();
1871            git::branch_delete(&repo, &branch).await.ok();
1872            self.state.candidates[i].folded = true;
1873            folded.push(label.to_string());
1874        }
1875        // The judges are finished; their checkouts are pure cost from here.
1876        let root = self.state.worktree_root();
1877        for j in 1..=self.roles.judges.len() {
1878            let wt = root.join(format!("judge-{j}"));
1879            if wt.exists() {
1880                git::worktree_remove(&repo, &wt).await.ok();
1881            }
1882        }
1883        if !folded.is_empty() {
1884            self.state
1885                .event("fold", format!("folded candidates {}", folded.join(", ")));
1886            self.state.save()?;
1887        }
1888        Ok(())
1889    }
1890
1891    // ------------------------------------------------------------ base sync
1892
1893    /// Land the winner's tree on the current tip of `<remote>/<base>` before
1894    /// anything verifies it.
1895    ///
1896    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
1897    /// read whatever is checked out in the winner's worktree. Left alone that
1898    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
1899    /// it when the run *branched* - and a run takes long enough that the base
1900    /// has usually moved by the time it gets here. A gate that ran there
1901    /// answers "green on the commit this run started from", not "green on
1902    /// what is about to land", and the difference showed up three times in
1903    /// one day as a green run whose merge would have reverted a file another
1904    /// pull request had already landed.
1905    ///
1906    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
1907    /// implementation of the same idea: `land::Step::Rebase` already worked
1908    /// out the rules - throwaway worktree, conflict stops and reports rather
1909    /// than feeding a fixer, nothing runs in the primary tree - and a second
1910    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
1911    /// warns about ("two answers to a question nobody notices until a diff is
1912    /// wrong").
1913    ///
1914    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
1915    /// so it survives a park/resume. A conflict or a push failure sets
1916    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
1917    /// as they were - untouched, for a person to look at - which is also what
1918    /// makes re-entering this function afterwards a no-op instead of a second
1919    /// attempt at the same wall.
1920    async fn sync_to_base(&mut self) -> Result<()> {
1921        if self
1922            .state
1923            .base_sync
1924            .as_ref()
1925            .is_some_and(|s| s.conflict.is_some())
1926        {
1927            return Ok(());
1928        }
1929        let Some(winner) = self.state.winner().cloned() else {
1930            return Ok(());
1931        };
1932
1933        let repo = self.state.repo.clone();
1934        let remote = self.state.config.merge.remote.clone();
1935        let base_branch = self.state.base_branch.clone();
1936        let tracking = format!("{remote}/{base_branch}");
1937
1938        git::fetch(&repo, &remote, &base_branch).await.ok();
1939        // No network, or the remote never had this branch: `resolve_base`
1940        // already treats that as non-fatal at branch time, and a run that got
1941        // this far must not be blocked by it here either.
1942        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
1943            return Ok(());
1944        };
1945
1946        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1947        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
1948        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
1949
1950        if behind == 0 {
1951            self.state.base_sync = Some(BaseSync {
1952                tip,
1953                behind: 0,
1954                attempts,
1955                conflict: None,
1956            });
1957            self.state.save()?;
1958            return Ok(());
1959        }
1960
1961        if attempts >= BASE_SYNC_ROUNDS {
1962            let why = format!(
1963                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
1964                 rebase(s); rebasing again would only race it",
1965                winner.branch
1966            );
1967            self.state.status = RunStatus::Blocked;
1968            self.state.base_sync = Some(BaseSync {
1969                tip,
1970                behind,
1971                attempts,
1972                conflict: Some(why.clone()),
1973            });
1974            self.state.event("land", why);
1975            self.state.save()?;
1976            return Ok(());
1977        }
1978
1979        self.state.event(
1980            "land",
1981            format!(
1982                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
1983                winner.branch
1984            ),
1985        );
1986        self.state.save()?;
1987
1988        let scratch = self.state.dir().join("base-sync");
1989        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
1990        let attempts = attempts + 1;
1991        match rebased {
1992            Ok(None) => {
1993                // The branch ref moved, but a worktree that already had it
1994                // checked out (the winner's) was not told; sync its index and
1995                // files before anything reads them.
1996                git::sync_to_head(&winner.worktree).await?;
1997                self.state.base_sync = Some(BaseSync {
1998                    tip: tip.clone(),
1999                    behind: 0,
2000                    attempts,
2001                    conflict: None,
2002                });
2003                self.state
2004                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
2005            }
2006            Ok(Some(conflict)) => {
2007                let why = format!(
2008                    "{} conflicts with {tracking} and did not rebase: {}",
2009                    winner.branch,
2010                    conflict.chars().take(600).collect::<String>()
2011                );
2012                self.state.status = RunStatus::Blocked;
2013                self.state.base_sync = Some(BaseSync {
2014                    tip,
2015                    behind,
2016                    attempts,
2017                    conflict: Some(why.clone()),
2018                });
2019                self.state.event("land", why);
2020            }
2021            Err(e) => {
2022                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
2023                self.state.status = RunStatus::Blocked;
2024                self.state.base_sync = Some(BaseSync {
2025                    tip,
2026                    behind,
2027                    attempts,
2028                    conflict: Some(why.clone()),
2029                });
2030                self.state.event("land", why);
2031            }
2032        }
2033        self.state.save()?;
2034        Ok(())
2035    }
2036
2037    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
2038    /// last landed the winner on, once it has run, else the commit the run
2039    /// branched from.
2040    ///
2041    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
2042    /// and `vote` all happen before there is a winner to rebase, so they
2043    /// compare every candidate against the branch point on purpose, and a
2044    /// base that moves after they are already done cannot change an answer
2045    /// they already gave.
2046    fn landing_base(&self) -> String {
2047        self.state
2048            .base_sync
2049            .as_ref()
2050            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2051    }
2052
2053    // --------------------------------------------------------------- review
2054
2055    async fn review_loop(&mut self) -> Result<()> {
2056        // A base that would not rebase is a person's decision, not a review
2057        // round: nothing here would change the answer, and reviewers and a
2058        // fixer would be spending real budget on a tree that cannot land
2059        // regardless of what they find.
2060        if self
2061            .state
2062            .base_sync
2063            .as_ref()
2064            .is_some_and(|s| s.conflict.is_some())
2065        {
2066            return Ok(());
2067        }
2068        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2069        // agent files with `magi task add` name the run that paid for it. The
2070        // prompt overlay is cloned alongside it because the waves borrow it
2071        // while `self` is mutably borrowed by the node's own bookkeeping.
2072        let run_id = self.state.id.clone();
2073        let prompts = self.state.config.prompts.clone();
2074        let Some(winner) = self.state.winner().cloned() else {
2075            return Ok(());
2076        };
2077        let max_rounds = self.state.config.graph.review_rounds;
2078        // A clean round, an exhausted round budget, or a stalled tree (see
2079        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
2080        // they are recorded — recomputed here, not read off `status`, so a
2081        // reentry into a run that already stopped restates the identical
2082        // verdict instead of silently handing back whatever an earlier node
2083        // in this same walk clobbered `status` to (a solo-candidate
2084        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
2085        // below runs an empty range once the budget is spent, and would
2086        // otherwise fall through without touching `status` at all.
2087        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
2088            self.state.status = status;
2089            self.state.save()?;
2090            return Ok(());
2091        }
2092        self.state.status = RunStatus::Reviewing;
2093
2094        let repo = self.state.repo.clone();
2095        let root = self.state.worktree_root();
2096        let language = self.state.config.graph.language.clone();
2097        let sessions = self.state.config.graph.sessions;
2098        let artifacts = agent::artifacts_dir(&self.state.dir());
2099        let base = self.landing_base();
2100        let base_short = short(&base);
2101        let reviewers = self.roles.reviewers.clone();
2102        let shell = self.state.config.shell();
2103
2104        let mut prev_e2e: Option<String> = None;
2105        for round in (self.state.reviews.len() + 1)..=max_rounds {
2106            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2107            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2108            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2109
2110            // Each reviewer gets its own detached checkout of exactly this
2111            // commit: nobody can perturb the winner's tree, and the fixer can
2112            // keep working without racing a reviewer.
2113            let mut jobs = Vec::new();
2114            for (r, spec) in reviewers.iter().cloned().enumerate() {
2115                let wt = root.join(format!("review-{}", r + 1));
2116                if wt.exists() {
2117                    git::reset_detached(&wt, &head).await?;
2118                } else {
2119                    git::worktree_add_detached(&repo, &wt, &head).await?;
2120                }
2121                let seat_key = format!("review-{}", r + 1);
2122                let seat = self.seat(&seat_key, &spec.id);
2123                jobs.push(SeatJob {
2124                    prompt: prompt::review(&prompt::ReviewCtx {
2125                        instruction: &self.state.instruction,
2126                        branch: &winner.branch,
2127                        base_short: &base_short,
2128                        stat: &stat,
2129                        patch: &patch,
2130                        e2e: prev_e2e.as_deref(),
2131                        reviewers: reviewers.len(),
2132                        round,
2133                        rounds: max_rounds,
2134                        // A review-only run has no rankings, so nothing
2135                        // competed for this patch and the reviewer is told so.
2136                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2137                        lens: Lens::for_seat(r),
2138                        language: &language,
2139                    }),
2140                    spec,
2141                    seat,
2142                    cwd: wt,
2143                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2144                    allow_write: false,
2145                    sessions,
2146                    artifacts: artifacts.clone(),
2147                    stem: format!("review-{round}-{}", r + 1),
2148                });
2149            }
2150
2151            self.state.event(
2152                "review",
2153                format!(
2154                    "round {round}: {} reviewers on {}",
2155                    jobs.len(),
2156                    short(&head)
2157                ),
2158            );
2159            let mut quota_losses = Vec::new();
2160            let review_retries = self.state.config.graph.retries;
2161            let review_cache = self.state.config.cache_dir();
2162            let ctx = WaveCtx {
2163                run: &run_id,
2164                node: "review",
2165                prompts: &prompts,
2166                cache: review_cache.as_deref(),
2167            };
2168            let results = ask_json_wave::<Review>(
2169                jobs,
2170                Arc::clone(&self.sem),
2171                review_retries,
2172                &ctx,
2173                &mut quota_losses,
2174                &mut self.state,
2175                &|_: &Review| Ok(()),
2176            )
2177            .await;
2178            self.state.quota.extend(quota_losses);
2179
2180            let mut records = Vec::new();
2181            let mut all_findings = Vec::new();
2182            for (r, (seat, res)) in results.into_iter().enumerate() {
2183                let agent_id = seat.agent.clone();
2184                self.state.seats.insert(seat.key.clone(), seat);
2185                let mut record = ReviewRecord {
2186                    reviewer: r + 1,
2187                    agent: agent_id,
2188                    summary: String::new(),
2189                    findings: Vec::new(),
2190                    vote: None,
2191                    failed: None,
2192                    duration_ms: 0,
2193                };
2194                match res {
2195                    Ok((review, out)) => {
2196                        // Sanitized here, at the point every other piece of
2197                        // agent prose in this file is (candidate summaries,
2198                        // deliberation turns, vote reasons): a reviewer's own
2199                        // words are the one thing about it that could name
2200                        // it, and reconsideration below broadcasts this same
2201                        // summary and these same findings to every other
2202                        // seat on the panel.
2203                        record.summary =
2204                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
2205                        record.vote = Some(review.vote);
2206                        record.duration_ms = out.duration_ms;
2207                        for (n, mut f) in review.findings.into_iter().enumerate() {
2208                            // ids are magi's, never the agent's: the fixer's
2209                            // adoption report is keyed by them.
2210                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2211                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
2212                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
2213                            // `file` is agent-supplied prose too, never
2214                            // checked against the real tree — the same
2215                            // exposure `title`/`detail` above have, just in
2216                            // a field easy to forget because it looks like a
2217                            // path rather than free text.
2218                            f.file = f
2219                                .file
2220                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
2221                            all_findings.push(f.clone());
2222                            record.findings.push(f);
2223                        }
2224                        self.state.event(
2225                            "review",
2226                            format!(
2227                                "round {round}: reviewer {} voted {} with {} finding(s)",
2228                                r + 1,
2229                                review.vote.label(),
2230                                record.findings.len()
2231                            ),
2232                        );
2233                    }
2234                    Err(e) => {
2235                        record.failed = Some(e.to_string());
2236                        self.state.event(
2237                            "review",
2238                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2239                        );
2240                    }
2241                }
2242                records.push(record);
2243            }
2244
2245            // Tally the round's votes and, if they split, spend the one
2246            // round of reconsideration the split -> deliberate -> revote
2247            // shape `judge`/`vote` use for the panel, sized down to what a
2248            // read-only review round can afford: one round, and a revote
2249            // rather than an argument, because the panel already wrote its
2250            // reasoning down as findings the first time around.
2251            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
2252            let vote_split =
2253                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
2254            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
2255            if vote_split {
2256                self.state.event(
2257                    "review",
2258                    format!(
2259                        "round {round}: votes split ({}) — one round of reconsideration",
2260                        initial_votes
2261                            .iter()
2262                            .map(|v| v.label())
2263                            .collect::<Vec<_>>()
2264                            .join(", ")
2265                    ),
2266                );
2267                // Seats read every seat's findings and votes, still numbered
2268                // and never named — the same anonymity `review` itself keeps.
2269                let panel: Vec<ReviewSeatReport<'_>> = records
2270                    .iter()
2271                    .filter_map(|r| {
2272                        r.vote.map(|vote| ReviewSeatReport {
2273                            reviewer: r.reviewer,
2274                            vote,
2275                            summary: &r.summary,
2276                            findings: &r.findings,
2277                        })
2278                    })
2279                    .collect();
2280
2281                let mut jobs = Vec::new();
2282                let mut seats_at = Vec::new();
2283                for (r, spec) in reviewers.iter().cloned().enumerate() {
2284                    // A seat with no initial vote has nothing to reconsider
2285                    // from and stays absent, the same as it stayed absent
2286                    // from `panel` above.
2287                    if records[r].vote.is_none() {
2288                        continue;
2289                    }
2290                    let wt = root.join(format!("review-{}", r + 1));
2291                    let seat_key = format!("review-{}", r + 1);
2292                    let seat = self.seat(&seat_key, &spec.id);
2293                    // A seat with no live session has already forgotten the
2294                    // initial review's prompt — restate the patch it is
2295                    // voting on, the same as `deliberate`/`vote` do for a
2296                    // judge in the same position.
2297                    let patch_ctx = if has_context(&spec, &seat, sessions) {
2298                        None
2299                    } else {
2300                        Some(ReviewPatch {
2301                            branch: &winner.branch,
2302                            base_short: &base_short,
2303                            stat: &stat,
2304                            patch: &patch,
2305                        })
2306                    };
2307                    let prompt = prompt::review_reconsider(&ReviewReconsiderCtx {
2308                        instruction: &self.state.instruction,
2309                        reviewer: r + 1,
2310                        lens: Lens::for_seat(r),
2311                        panel: &panel,
2312                        patch: patch_ctx,
2313                        round,
2314                        rounds: max_rounds,
2315                        language: &language,
2316                    });
2317                    jobs.push(SeatJob {
2318                        prompt,
2319                        spec,
2320                        seat,
2321                        cwd: wt,
2322                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2323                        allow_write: false,
2324                        sessions,
2325                        artifacts: artifacts.clone(),
2326                        stem: format!("review-{round}-reconsider-{}", r + 1),
2327                    });
2328                    seats_at.push(r);
2329                }
2330
2331                let mut recon_quota_losses = Vec::new();
2332                let recon_cache = self.state.config.cache_dir();
2333                let recon_ctx = WaveCtx {
2334                    run: &run_id,
2335                    node: "review",
2336                    prompts: &prompts,
2337                    cache: recon_cache.as_deref(),
2338                };
2339                let recon_results = ask_json_wave::<ReviewRevote>(
2340                    jobs,
2341                    Arc::clone(&self.sem),
2342                    review_retries,
2343                    &recon_ctx,
2344                    &mut recon_quota_losses,
2345                    &mut self.state,
2346                    &|_: &ReviewRevote| Ok(()),
2347                )
2348                .await;
2349                self.state.quota.extend(recon_quota_losses);
2350
2351                for (&r, (seat, res)) in seats_at.iter().zip(recon_results) {
2352                    let agent_id = seat.agent.clone();
2353                    self.state.seats.insert(seat.key.clone(), seat);
2354                    let mut rec = ReviewRevoteRecord {
2355                        reviewer: r + 1,
2356                        agent: agent_id,
2357                        vote: None,
2358                        reason: String::new(),
2359                        failed: None,
2360                    };
2361                    match res {
2362                        Ok((rv, _)) => {
2363                            rec.vote = Some(rv.vote);
2364                            rec.reason =
2365                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
2366                            self.state.event(
2367                                "review",
2368                                format!(
2369                                    "round {round}: reviewer {} revoted {}",
2370                                    r + 1,
2371                                    rv.vote.label()
2372                                ),
2373                            );
2374                        }
2375                        Err(e) => {
2376                            rec.failed = Some(e.to_string());
2377                            self.state.event(
2378                                "review",
2379                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
2380                            );
2381                        }
2382                    }
2383                    reconsideration.push(rec);
2384                }
2385            } else if initial_votes.len() > 1 {
2386                self.state.event(
2387                    "review",
2388                    format!(
2389                        "round {round}: votes agreed ({}) — no reconsideration",
2390                        initial_votes[0].label()
2391                    ),
2392                );
2393            }
2394
2395            // The final vote per seat is its revote where reconsideration
2396            // ran and answered, its initial vote otherwise — the same
2397            // fallback `tally` uses for a judge whose private vote failed.
2398            let final_votes: Vec<ReviewVote> = records
2399                .iter()
2400                .filter_map(|r| {
2401                    reconsideration
2402                        .iter()
2403                        .find(|rv| rv.reviewer == r.reviewer)
2404                        .and_then(|rv| rv.vote)
2405                        .or(r.vote)
2406                })
2407                .collect();
2408            let round_verdict = ReviewVote::worst(final_votes);
2409
2410            let mut e2e = run_commands(
2411                &shell,
2412                &self.state.config.verify.e2e,
2413                &winner.worktree,
2414                Duration::from_secs(self.state.config.graph.timeout_review),
2415            )
2416            .await;
2417            for o in &e2e {
2418                self.state.event(
2419                    "verify",
2420                    format!("round {round}: `{}` -> {}", o.command, e2e_outcome_label(o)),
2421                );
2422            }
2423
2424            // A build/link failure is not a verdict on the patch — it is
2425            // frequently a race against a shared `CARGO_TARGET_DIR` (see
2426            // AGENTS.md). Give verify one retry before letting a red like
2427            // that decide the round.
2428            let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
2429            if verify_retried {
2430                self.state.event(
2431                    "verify",
2432                    format!(
2433                        "round {round}: verify could not build/link, not a test result — \
2434                         retrying once before concluding"
2435                    ),
2436                );
2437                e2e = run_commands(
2438                    &shell,
2439                    &self.state.config.verify.e2e,
2440                    &winner.worktree,
2441                    Duration::from_secs(self.state.config.graph.timeout_review),
2442                )
2443                .await;
2444                for o in &e2e {
2445                    self.state.event(
2446                        "verify",
2447                        format!(
2448                            "round {round}: retry `{}` -> {}",
2449                            o.command,
2450                            e2e_outcome_label(o)
2451                        ),
2452                    );
2453                }
2454            }
2455
2456            let e2e_failures: String = e2e
2457                .iter()
2458                .filter(|o| !o.ok())
2459                .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2460                .collect();
2461
2462            let expected = records.len();
2463            let answered = records.iter().filter(|r| r.failed.is_none()).count();
2464            let incomplete = answered < expected;
2465            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2466            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
2467            let policy = self.state.config.graph.incomplete_review;
2468            let clean = round_is_clean(blocking, e2e_ok, answered, expected, policy);
2469
2470            let mut round_record = ReviewRound {
2471                round,
2472                head: head.clone(),
2473                reviews: records,
2474                e2e,
2475                verify_retried,
2476                fix: None,
2477                blocking,
2478                answered,
2479                expected,
2480                clean,
2481                progressed: false,
2482                vote_split,
2483                reconsideration,
2484                verdict: round_verdict,
2485            };
2486
2487            if incomplete {
2488                let missing: Vec<String> = round_record
2489                    .reviews
2490                    .iter()
2491                    .filter(|r| r.failed.is_some())
2492                    .map(|r| format!("review-{}", r.reviewer))
2493                    .collect();
2494                self.state.event(
2495                    "review",
2496                    format!(
2497                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
2498                        missing.join(", ")
2499                    ),
2500                );
2501            }
2502
2503            if clean {
2504                self.state.event(
2505                    "review",
2506                    if incomplete {
2507                        format!(
2508                            "round {round}: clean (warn policy, incomplete panel) — no \
2509                             blocking findings from the seats that answered, verification green"
2510                        )
2511                    } else {
2512                        format!("round {round}: clean — no blocking findings, verification green")
2513                    },
2514                );
2515                self.state.reviews.push(round_record);
2516                self.state.status = RunStatus::Gating;
2517                self.state.save()?;
2518                return Ok(());
2519            }
2520
2521            // Nothing was raised and verification passed, but not every seat
2522            // answered and the policy refuses to call that clean: re-review
2523            // rather than send the fixer after a round with nothing to fix.
2524            if incomplete && blocking == 0 && e2e_ok {
2525                self.state.reviews.push(round_record);
2526                self.state.save()?;
2527                if round == max_rounds {
2528                    self.state.status = RunStatus::Blocked;
2529                    self.state.event(
2530                        "review",
2531                        format!(
2532                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
2533                             refusing to call it clean",
2534                            expected - answered
2535                        ),
2536                    );
2537                    return Ok(());
2538                }
2539                prev_e2e = None;
2540                continue;
2541            }
2542
2543            if round == max_rounds {
2544                self.state.reviews.push(round_record);
2545                return self.stop_reviewing(&format!(
2546                    "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
2547                ));
2548            }
2549
2550            // Fix. The winner's own implementer seat continues its conversation:
2551            // the competition is over, so context is pure benefit now.
2552            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
2553                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
2554                _ => (
2555                    self.state
2556                        .config
2557                        .agent(&winner.agent)
2558                        .cloned()
2559                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
2560                    format!("impl-{}", winner.label),
2561                ),
2562            };
2563            let seat = self.seat(&fix_seat_key, &fix_spec.id);
2564            let blocking_findings: Vec<_> = all_findings
2565                .iter()
2566                .filter(|f| f.severity.blocks())
2567                .cloned()
2568                .collect();
2569            let job = SeatJob {
2570                prompt: prompt::fix(
2571                    &self.state.instruction,
2572                    &blocking_findings,
2573                    (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
2574                    round,
2575                    max_rounds,
2576                    &language,
2577                ),
2578                spec: fix_spec.clone(),
2579                seat,
2580                cwd: winner.worktree.clone(),
2581                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
2582                allow_write: true,
2583                sessions,
2584                artifacts: artifacts.clone(),
2585                stem: format!("fix-{round}"),
2586            };
2587            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
2588            let cache = self.state.config.cache_dir();
2589            let ctx = WaveCtx {
2590                run: &run_id,
2591                node: "fix",
2592                prompts: &prompts,
2593                cache: cache.as_deref(),
2594            };
2595            let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2596            let agent_id = seat.agent.clone();
2597            let seat_key = seat.key.clone();
2598            self.state.seats.insert(seat.key.clone(), seat);
2599
2600            let mut fix = FixRecord {
2601                agent: agent_id,
2602                addressed: Vec::new(),
2603                rejected: Vec::new(),
2604                notes: String::new(),
2605                committed: false,
2606                failed: None,
2607                duration_ms: 0,
2608            };
2609            match out {
2610                AgentOutcome::Ok(o) => {
2611                    fix.duration_ms = o.duration_ms;
2612                    match verdict::extract_json::<FixReport>(&o.text) {
2613                        Ok(report) => {
2614                            fix.addressed = report.addressed;
2615                            fix.rejected = report.rejected;
2616                            fix.notes =
2617                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
2618                        }
2619                        Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
2620                    }
2621                }
2622                // The CLI's raw error JSON is not a fix report to parse.
2623                AgentOutcome::Dropped(o) => {
2624                    fix.duration_ms = o.duration_ms;
2625                    let why = o
2626                        .dropped
2627                        .as_ref()
2628                        .map(|d| d.why.as_str())
2629                        .unwrap_or("the CLI ended the stream without delivering its answer");
2630                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
2631                }
2632                AgentOutcome::Quota(o) => {
2633                    self.state.quota.push(QuotaLoss {
2634                        seat: seat_key,
2635                        node: "fix".to_owned(),
2636                        at: Timestamp::now(),
2637                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2638                    });
2639                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
2640                }
2641                AgentOutcome::Failed(e) => fix.failed = Some(e),
2642            }
2643            git::commit_all(
2644                &winner.worktree,
2645                &format!("magi: review round {round} fixes (uncommitted work)"),
2646            )
2647            .await
2648            .ok();
2649            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
2650            fix.committed = after != before;
2651            // Judged by what `git` says moved against base, never by the
2652            // fixer's own `addressed`/`rejected` count — see
2653            // `ReviewRound::progressed`. Propagated with `?`, the same as the
2654            // `patch` snapshot above: swallowing this error would default
2655            // `diff_after` to empty, which almost always differs from a
2656            // non-empty `patch` and reads as "progressed" — exactly backwards
2657            // for a `git` failure the stagnation check cannot see through.
2658            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
2659            let progressed = diff_after != patch;
2660            let commit_note = if fix.committed {
2661                "committed"
2662            } else {
2663                "NO new commit"
2664            };
2665            let tree_note = if progressed {
2666                "changed vs base"
2667            } else {
2668                "unchanged vs base"
2669            };
2670            self.state.event(
2671                "fix",
2672                match &fix.failed {
2673                    // Distinct on purpose from "0 addressed, 0 rejected": the
2674                    // fixer's own diff still landed (blocking counts do keep
2675                    // falling round over round), only its adoption report did
2676                    // not come back, so this must never read like every
2677                    // finding was reviewed and declined.
2678                    Some(reason) => {
2679                        format!(
2680                            "round {round}: fixer's adoption report was lost ({reason}); \
2681                             {commit_note}, tree {tree_note}"
2682                        )
2683                    }
2684                    None => format!(
2685                        "round {round}: {} addressed, {} rejected, {commit_note}, tree {tree_note}",
2686                        fix.addressed.len(),
2687                        fix.rejected.len(),
2688                    ),
2689                },
2690            );
2691            round_record.fix = Some(fix);
2692            round_record.progressed = progressed;
2693            self.state.reviews.push(round_record);
2694            self.state.save()?;
2695
2696            prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
2697
2698            let streak = self
2699                .state
2700                .reviews
2701                .iter()
2702                .rev()
2703                .take_while(|r| !r.progressed)
2704                .count();
2705            if streak >= STAGNANT_LIMIT {
2706                return self.stop_reviewing(&format!(
2707                    "the tree has not moved against base for {streak} round(s) in a row"
2708                ));
2709            }
2710        }
2711        Ok(())
2712    }
2713
2714    /// Decide, from the last recorded round's own verification, whether
2715    /// stopping the review loop is a hand-off or a genuine block.
2716    ///
2717    /// Called once the loop has given up trying — the round budget is spent,
2718    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
2719    /// findings still open, never while a round is still clean or the
2720    /// incomplete-panel case handled inline above. Gate and e2e are facts
2721    /// about the tree; a lingering review finding is an opinion, and this
2722    /// workload's own `magi stats` puts reviewer precision low enough
2723    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
2724    /// must not by itself stand between a green, verified change and the
2725    /// human who decides what to do with it. A red e2e is not an opinion, so
2726    /// that case still blocks, with the failing command and a tail of its
2727    /// output recorded here rather than left in `run.json` for someone to go
2728    /// find.
2729    fn stop_reviewing(&mut self, why: &str) -> Result<()> {
2730        let last = self
2731            .state
2732            .reviews
2733            .last()
2734            .expect("a round was just recorded before this is called");
2735        let red: Vec<String> = last
2736            .e2e
2737            .iter()
2738            .filter(|o| !o.ok())
2739            .map(|o| {
2740                format!(
2741                    "`{}` -> {:?}\n{}",
2742                    o.command,
2743                    o.code,
2744                    tail(&o.output_tail, EVENT_OUTPUT_TAIL)
2745                )
2746            })
2747            .collect();
2748        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
2749
2750        if red.is_empty() {
2751            self.state.event(
2752                "review",
2753                format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
2754            );
2755            self.state.status = RunStatus::Gating;
2756        } else {
2757            self.state
2758                .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
2759            self.state.status = RunStatus::Blocked;
2760        }
2761        self.state.save()?;
2762        Ok(())
2763    }
2764
2765    // ----------------------------------------------------------------- gate
2766
2767    async fn gate(&mut self) -> Result<()> {
2768        // Judged by the review record itself, not by `status`: a solo
2769        // candidate's `judge`/`deliberate` skip rewrites `status` on every
2770        // reentry (see `judge`), and trusting it here is exactly how a run
2771        // that exhausted its review budget got gated and merged a second
2772        // time around. `review_conclusion` recomputes the review loop's own
2773        // verdict from the round records themselves — `Gating` for a clean
2774        // round or a hand-off (see `stop_reviewing`), anything else means the
2775        // loop is still going or genuinely blocked.
2776        // A base the winner could not be replayed onto is a decision, not a
2777        // round: there is no landing tree to gate. Read as its own record for
2778        // the same reason the review verdict is.
2779        if self.state.status == RunStatus::Failed
2780            || self
2781                .state
2782                .base_sync
2783                .as_ref()
2784                .is_some_and(|s| s.conflict.is_some())
2785            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
2786                != Some(RunStatus::Gating)
2787        {
2788            return Ok(());
2789        }
2790        if !self.state.gate.is_empty() {
2791            return Ok(());
2792        }
2793        let Some(winner) = self.state.winner().cloned() else {
2794            return Ok(());
2795        };
2796        self.state.status = RunStatus::Gating;
2797        let shell = self.state.config.shell();
2798        let outcomes = run_commands(
2799            &shell,
2800            &self.state.config.verify.gate,
2801            &winner.worktree,
2802            Duration::from_secs(self.state.config.graph.timeout_review),
2803        )
2804        .await;
2805        for o in &outcomes {
2806            self.state.event(
2807                "gate",
2808                format!(
2809                    "`{}` -> {}",
2810                    o.command,
2811                    if o.ok() {
2812                        "pass".to_owned()
2813                    } else {
2814                        format!(
2815                            "FAIL ({:?})\n{}",
2816                            o.code,
2817                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
2818                        )
2819                    }
2820                ),
2821            );
2822        }
2823        let passed = outcomes.iter().all(CommandOutcome::ok);
2824        self.state.gate = outcomes;
2825        if !passed {
2826            self.state.status = RunStatus::Blocked;
2827            self.state.event("gate", "gate failed; not merging");
2828        }
2829        self.state.save()?;
2830        Ok(())
2831    }
2832
2833    // ---------------------------------------------------------------- merge
2834
2835    async fn merge(&mut self) -> Result<()> {
2836        // Same reasoning as `gate`: ask the review and gate records directly
2837        // rather than `status`, which a solo-candidate `judge`/`deliberate`
2838        // skip can rewrite on reentry to something that no longer says
2839        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
2840        // so a hand-off (open findings, green verification) reaches merge
2841        // exactly like a genuinely clean round does.
2842        //
2843        // A run resumed mid-`land` never reaches here at all: `execute`
2844        // recognises `RunStatus::Landing` before it even calls `prep`, and
2845        // routes straight to `run_land` instead. That has to happen a level
2846        // up from this function, not with a check in here, because
2847        // `review_loop`'s own status recomputation (see its doc) runs
2848        // *before* `merge` on every reentry and would otherwise overwrite
2849        // the `Landing` marker with `Gating` before this node ever saw it.
2850        if self
2851            .state
2852            .base_sync
2853            .as_ref()
2854            .is_some_and(|s| s.conflict.is_some())
2855            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
2856                != Some(RunStatus::Gating)
2857            || self.state.gate.iter().any(|o| !o.ok())
2858        {
2859            return Ok(());
2860        }
2861        // This node's own record, not `status`: `status == Ready` is not
2862        // unique to the harmless `MergeMode::None` path this line was
2863        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
2864        // run's PR was closed without merging — and on that run `mode` is
2865        // still `Pr`, so a reentry that fell through here would push and
2866        // open a second pull request. `self.state.merge` is set exactly once
2867        // this node (or `land`) has already produced a verdict, under every
2868        // mode, which is what "already done" actually means here.
2869        if self.state.merge.is_some() {
2870            return Ok(());
2871        }
2872        let Some(winner) = self.state.winner().cloned() else {
2873            return Ok(());
2874        };
2875        let repo = self.state.repo.clone();
2876        let base = self.state.base_branch.clone();
2877        let mode = self.state.config.merge.mode;
2878        let message = pr_body(&self.state, winner.label);
2879
2880        let outcome = match mode {
2881            MergeMode::None => MergeOutcome {
2882                mode,
2883                ok: true,
2884                detail: format!("git -C {} merge --no-ff {}", repo.display(), winner.branch),
2885            },
2886            MergeMode::Local => {
2887                let on = git::current_branch(&repo).await?;
2888                if on.as_deref() != Some(base.as_str()) {
2889                    MergeOutcome {
2890                        mode,
2891                        ok: false,
2892                        detail: format!(
2893                            "{} has {} checked out, not the base branch {base}",
2894                            repo.display(),
2895                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
2896                        ),
2897                    }
2898                } else if !git::is_clean(&repo).await? {
2899                    MergeOutcome {
2900                        mode,
2901                        ok: false,
2902                        detail: format!("{} is dirty; refusing to merge", repo.display()),
2903                    }
2904                } else {
2905                    let out = git::merge_no_ff(&repo, &winner.branch, &message).await?;
2906                    MergeOutcome {
2907                        mode,
2908                        ok: out.ok(),
2909                        detail: if out.ok() { out.stdout } else { out.stderr },
2910                    }
2911                }
2912            }
2913            MergeMode::Pr => {
2914                let remote = self.state.config.merge.remote.clone();
2915                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
2916                if !pushed.ok() {
2917                    MergeOutcome {
2918                        mode,
2919                        ok: false,
2920                        detail: pushed.stderr,
2921                    }
2922                } else {
2923                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
2924                    match out {
2925                        Ok(url) => MergeOutcome {
2926                            mode,
2927                            ok: true,
2928                            detail: url,
2929                        },
2930                        Err(e) => MergeOutcome {
2931                            mode,
2932                            ok: false,
2933                            detail: e.to_string(),
2934                        },
2935                    }
2936                }
2937            }
2938        };
2939
2940        self.state.status = match (mode, outcome.ok) {
2941            (MergeMode::None, _) => RunStatus::Ready,
2942            (_, true) => RunStatus::Merged,
2943            (_, false) => RunStatus::Blocked,
2944        };
2945        self.state.event(
2946            "merge",
2947            format!(
2948                "{:?}: {}",
2949                mode,
2950                outcome.detail.lines().next().unwrap_or("")
2951            ),
2952        );
2953        self.state.merge = Some(outcome);
2954        self.state.save()?;
2955
2956        // The PR is open and the run would historically stop here, leaving the
2957        // operator to watch checks, feed review comments back to a fixer, and
2958        // merge. That was done by hand six times in one session before this
2959        // existed. Opt-in, because merging is the one irreversible thing magi
2960        // can do to a repository.
2961        if self.state.config.graph.land
2962            && mode == MergeMode::Pr
2963            && self.state.status == RunStatus::Merged
2964        {
2965            self.run_land().await?;
2966        }
2967        Ok(())
2968    }
2969
2970    /// Enter `land`.
2971    ///
2972    /// Shared between a fresh run's first pass through [`Runner::merge`] and
2973    /// a resumed run's re-entry. `land::land` itself is what serialises the
2974    /// two git-mutating moments inside the loop — the rebase push and
2975    /// `gh pr merge` — per repository (see its own doc); nothing here needs
2976    /// to hold a lock across the whole call, and doing so would serialise
2977    /// this run's CI wait against a *different* run's land-approval resume
2978    /// in the same repository, which is exactly the "must not wait on
2979    /// another task" property the daemon's slot-freeing exists to give.
2980    async fn run_land(&mut self) -> Result<()> {
2981        let url = self
2982            .state
2983            .merge
2984            .as_ref()
2985            .map(|m| m.detail.clone())
2986            .unwrap_or_default();
2987        let url = url.lines().next().unwrap_or("").trim().to_owned();
2988        if !url.starts_with("http") {
2989            return Ok(());
2990        }
2991        // A land failure is not a lost run: the work is on a branch and the
2992        // pull request is open, which is exactly where a human takes over.
2993        match land::land(&mut self.state, &url).await {
2994            Ok(pr) if self.state.parked => {
2995                // `land` already saved the parked marker; nothing here
2996                // overrides `status` back to a terminal value while an
2997                // approval is still outstanding.
2998                let _ = pr;
2999            }
3000            Ok(pr) => {
3001                self.state.status = match pr.state {
3002                    land::PrLifecycle::Merged => RunStatus::Merged,
3003                    _ => RunStatus::Blocked,
3004                };
3005                // Downstream of a confirmed merge only - see
3006                // `bump::should_release_bump`'s own doc for why this one
3007                // check covers all three of `land`'s success paths.
3008                // Best-effort: the run already landed, so a failure here
3009                // (the decision call, `gh`, `cargo`) is recorded and never
3010                // turns a landed run into a failed one.
3011                if bump::should_release_bump(self.state.status)
3012                    && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
3013                {
3014                    self.state
3015                        .event("bump", format!("release bump skipped: {e:#}"));
3016                }
3017                self.state.save()?;
3018            }
3019            Err(e) => {
3020                self.state.status = RunStatus::Blocked;
3021                self.state.event("land", format!("gave up: {e}"));
3022                self.state.save()?;
3023            }
3024        }
3025        Ok(())
3026    }
3027
3028    // -------------------------------------------------------------- helpers
3029
3030    /// Fetch or create a seat, keeping its conversation across nodes.
3031    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
3032        if let Some(existing) = self.state.seats.get(key)
3033            && existing.agent == agent
3034        {
3035            return existing.clone();
3036        }
3037        let fresh = SeatState::new(key, agent, self.state.seed);
3038        self.state.seats.insert(key.to_owned(), fresh.clone());
3039        fresh
3040    }
3041
3042    /// A candidate rendered for judging, with the leak policy applied.
3043    fn view(&self, c: &Candidate) -> CandidateView {
3044        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3045            .unwrap_or_default();
3046        let (patch, _) = blind::sanitize_patch(
3047            &format!("candidate {} patch", c.label),
3048            &raw,
3049            &self.state.config.blind,
3050        );
3051        CandidateView {
3052            label: c.label,
3053            branch: c.branch.clone(),
3054            summary: c.summary.clone(),
3055            stat: c.stat.clone(),
3056            patch,
3057        }
3058    }
3059
3060    /// The full candidate set as prompt text, for seats with no live session.
3061    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
3062        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
3063        prompt::judge(
3064            "(see above)",
3065            &views,
3066            self.roles.judges.len(),
3067            base_short,
3068            "en",
3069        )
3070    }
3071
3072    /// Anonymised transcript for judge `self_idx`.
3073    ///
3074    /// The initial rankings are always the opening statements. Seeding them
3075    /// only when no turn had been taken yet meant every judge after the first
3076    /// argued against a single voice instead of against the actual split — the
3077    /// disagreement is the information, so it is always on the table.
3078    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
3079        let mut turns = Vec::new();
3080        for j in &self.state.judgements {
3081            if j.ranking.is_empty() {
3082                continue;
3083            }
3084            let reasons = j
3085                .reasons
3086                .iter()
3087                .map(|(k, v)| format!("- {k}: {v}"))
3088                .collect::<Vec<_>>()
3089                .join("\n");
3090            turns.push(Turn {
3091                who: format!("Judge {} (opening ranking)", j.judge),
3092                is_self: j.judge == self_idx + 1,
3093                body: format!(
3094                    "Ranked {}{}{reasons}",
3095                    j.ranking.iter().collect::<String>(),
3096                    if reasons.is_empty() {
3097                        ""
3098                    } else {
3099                        ", because:\n"
3100                    }
3101                ),
3102            });
3103        }
3104        for t in self
3105            .state
3106            .deliberation
3107            .iter()
3108            .flat_map(|r| r.turns.iter())
3109            .chain(current)
3110        {
3111            turns.push(Turn {
3112                who: format!("Judge {}", t.judge),
3113                is_self: t.judge == self_idx + 1,
3114                body: t.body.clone(),
3115            });
3116        }
3117        turns
3118    }
3119}
3120
3121/// Does this seat still hold the context a follow-up prompt would rely on?
3122fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
3123    agent::has_session(spec.kind, seat, sessions)
3124}
3125
3126fn short(commit: &str) -> String {
3127    commit.chars().take(7).collect()
3128}
3129
3130fn make_executable(path: &Path) -> Result<()> {
3131    #[cfg(unix)]
3132    {
3133        use std::os::unix::fs::PermissionsExt as _;
3134        let mut perms = std::fs::metadata(path)?.permissions();
3135        perms.set_mode(0o755);
3136        std::fs::set_permissions(path, perms)?;
3137    }
3138    #[cfg(not(unix))]
3139    {
3140        let _ = path;
3141    }
3142    Ok(())
3143}
3144
3145/// What every seat in one batch shares: where the answers are attributed, the
3146/// prompt overlay they inherit, and the build cache they are told to use.
3147///
3148/// A struct rather than four more parameters: `wave` also needs the run's
3149/// state (to record who is answering right now) and the attempt number, and
3150/// eight positional arguments is both unreadable and a clippy error.
3151struct WaveCtx<'a> {
3152    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
3153    /// paid for it.
3154    run: &'a str,
3155    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
3156    node: &'a str,
3157    prompts: &'a Prompts,
3158    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
3159    cache: Option<&'a Path>,
3160}
3161
3162/// Run one job, honouring the parallelism budget.
3163async fn run_one(
3164    job: SeatJob,
3165    sem: Arc<Semaphore>,
3166    ctx: &WaveCtx<'_>,
3167    state: &mut RunState,
3168    attempt: usize,
3169) -> (SeatState, AgentOutcome) {
3170    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
3171        .await
3172        .pop()
3173        .expect("one job in, one result out");
3174    (seat, out)
3175}
3176
3177/// Run every job concurrently, capped by the semaphore, preserving order.
3178///
3179/// Every seat in the batch is recorded into [`RunState::active`] before the
3180/// wave starts and cleared as each answer lands, so the run's own record says
3181/// who is still being waited on rather than only who finished.
3182async fn wave(
3183    jobs: Vec<SeatJob>,
3184    sem: Arc<Semaphore>,
3185    ctx: &WaveCtx<'_>,
3186    state: &mut RunState,
3187    attempt: usize,
3188) -> Vec<(usize, SeatState, AgentOutcome)> {
3189    let WaveCtx {
3190        run,
3191        node,
3192        prompts,
3193        cache,
3194    } = *ctx;
3195    for job in &jobs {
3196        state.seat_started(node, &job.seat.key, job.timeout, attempt);
3197    }
3198    if let Err(e) = state.save() {
3199        // A failed persist of "who is answering right now" must not abort the
3200        // wave: the seats are already being asked, and the alternative is
3201        // losing the answers to save a status line nobody may even be
3202        // watching.
3203        tracing::warn!("could not persist in-progress seats: {e:#}");
3204    }
3205    let mut set = tokio::task::JoinSet::new();
3206    let overlay = prompts.overlay(node);
3207    for (i, mut job) in jobs.into_iter().enumerate() {
3208        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
3209        if cache.is_some() {
3210            job.prompt.push('\n');
3211            job.prompt.push_str(prompt::build_cache_note());
3212        }
3213        let sem = Arc::clone(&sem);
3214        let run = run.to_owned();
3215        let node = node.to_owned();
3216        let cache = cache.map(Path::to_path_buf);
3217        set.spawn(async move {
3218            let _permit = sem.acquire().await;
3219            let mut seat = job.seat;
3220            let out = agent::invoke(
3221                &job.spec,
3222                &mut seat,
3223                &Invocation {
3224                    cwd: &job.cwd,
3225                    prompt: &job.prompt,
3226                    timeout: job.timeout,
3227                    allow_write: job.allow_write,
3228                    sessions: job.sessions,
3229                    artifacts: &job.artifacts,
3230                    stem: &job.stem,
3231                    run: &run,
3232                    node: &node,
3233                    cache_dir: cache.as_deref(),
3234                },
3235            )
3236            .await;
3237            let out = match out {
3238                Ok(o) if o.usable() => AgentOutcome::Ok(o),
3239                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
3240                // Billed work the CLI failed to hand over is not an ordinary
3241                // failure, but its text is the CLI's raw error JSON, not an
3242                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
3243                // read it as one by forgetting to check. `usable()` is always
3244                // false here (dropped implies an empty response), so this has
3245                // to be checked before the catch-all `Failed` below or the
3246                // one shape this exists for is lost with the rest.
3247                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
3248                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
3249                Ok(o) => AgentOutcome::Failed(format!(
3250                    "exited with {:?} and no usable output",
3251                    o.exit_code
3252                )),
3253                Err(e) => AgentOutcome::Failed(e.to_string()),
3254            };
3255            (i, seat, out)
3256        });
3257    }
3258    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
3259    while let Some(joined) = set.join_next().await {
3260        let (i, seat, out) = match joined {
3261            Ok(v) => v,
3262            // No seat to clear: a panicked task never reported which one it
3263            // was. The defensive sweep below this loop is what stops that
3264            // seat's `active` entry from surviving forever.
3265            Err(e) => {
3266                tracing::error!("agent task panicked: {e}");
3267                continue;
3268            }
3269        };
3270        state.seat_finished(&seat.key);
3271        if let Err(e) = state.save() {
3272            tracing::warn!("could not persist a seat's completion: {e:#}");
3273        }
3274        if collected.len() <= i {
3275            collected.resize_with(i + 1, || None);
3276        }
3277        collected[i] = Some((i, seat, out));
3278    }
3279    // Belt-and-braces for the panic branch above: every seat this exact batch
3280    // started shares this `(node, attempt)` pair, and every seat that finished
3281    // normally already cleared itself, so anything left tagged with it here
3282    // can only be a panicked task's leftover. Cleared unconditionally rather
3283    // than left to read as still answering forever.
3284    if state
3285        .active
3286        .values()
3287        .any(|a| a.node == node && a.attempt == attempt)
3288    {
3289        state
3290            .active
3291            .retain(|_, a| !(a.node == node && a.attempt == attempt));
3292        if let Err(e) = state.save() {
3293            tracing::warn!("could not persist the end of a wave: {e:#}");
3294        }
3295    }
3296    collected.into_iter().flatten().collect()
3297}
3298
3299/// Is a review round clean, given how many reviewer seats answered against
3300/// how many the round expected?
3301///
3302/// A seat that never answered (timeout, crash, unparsable output) is not a
3303/// seat that read the patch and found nothing — treating it as such is
3304/// exactly the bug this function exists to close. Under the default `block`
3305/// policy a missing seat can never be clean; `warn` still requires the seats
3306/// that *did* answer to have found nothing blocking and verification to be
3307/// green.
3308fn round_is_clean(
3309    blocking: usize,
3310    e2e_ok: bool,
3311    answered: usize,
3312    expected: usize,
3313    policy: IncompleteReviewPolicy,
3314) -> bool {
3315    blocking == 0 && e2e_ok && (answered == expected || policy == IncompleteReviewPolicy::Warn)
3316}
3317
3318/// The review loop's own conclusion, derived entirely from its persisted
3319/// round records and the round budget that produced them — never from
3320/// `status`, so a reentry (or `gate`/`merge` reading it independently)
3321/// recomputes the identical answer regardless of what an earlier node in the
3322/// same walk, or a previous walk, did to `status`.
3323///
3324/// `None` while more rounds remain to try, including when review never ran
3325/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
3326/// gone clean, or the budget is spent, or the tree has stopped moving (see
3327/// [`STAGNANT_LIMIT`]), the answer is one of two things:
3328///
3329/// - An incomplete panel that raised nothing is missing input, not a
3330///   verified tree — never a hand-off candidate, whatever verification said
3331///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
3332/// - Otherwise, green e2e on the last round hands off (see
3333///   [`Runner::stop_reviewing`]); red e2e blocks.
3334fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
3335    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
3336        return Some(RunStatus::Gating);
3337    }
3338    let last = reviews.last()?;
3339    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
3340    if reviews.len() < max_rounds && !stagnant {
3341        return None;
3342    }
3343    Some(if last.incomplete() && last.blocking == 0 {
3344        RunStatus::Blocked
3345    } else if last.e2e.iter().all(CommandOutcome::ok) {
3346        RunStatus::Gating
3347    } else {
3348        RunStatus::Blocked
3349    })
3350}
3351
3352/// How long a re-ask may take, given the budget the first attempt had.
3353///
3354/// A `nudged` retry is a request to restate an answer the seat has already
3355/// worked out: it carries no new work, so it does not deserve the original
3356/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
3357/// 133 seconds while a third sat for over ten minutes on a resumed session
3358/// holding 410 KB of prior output - and because the retry had inherited the
3359/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
3360/// judging round whose other seats were long finished.
3361///
3362/// A quarter of the budget, with a floor so that a deliberately short timeout
3363/// does not collapse to nothing. A retry that re-sends the whole prompt
3364/// (because the seat kept no context) is the original job again, and keeps the
3365/// original budget.
3366fn retry_budget(full: Duration, nudged: bool) -> Duration {
3367    if nudged {
3368        (full / 4).max(Duration::from_secs(120)).min(full)
3369    } else {
3370        full
3371    }
3372}
3373
3374/// Run a wave and parse each reply, re-asking the seats whose reply was
3375/// unusable.
3376///
3377/// The re-ask is a nudge rather than the whole prompt again when the seat still
3378/// holds its conversation, which is the difference between a cheap retry and
3379/// paying for the entire candidate set twice.
3380///
3381/// A seat that hits a rate limit is **not** re-asked: the same call will fail
3382/// the same way until the limit resets, so spending a retry attempt on it is
3383/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
3384/// like any other absent seat — the caller decides whether the panel still has
3385/// a quorum.
3386#[allow(clippy::too_many_arguments)]
3387async fn ask_json_wave<T>(
3388    jobs: Vec<SeatJob>,
3389    sem: Arc<Semaphore>,
3390    retries: usize,
3391    ctx: &WaveCtx<'_>,
3392    losses: &mut Vec<QuotaLoss>,
3393    state: &mut RunState,
3394    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
3395) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
3396where
3397    T: serde::de::DeserializeOwned + Send + 'static,
3398{
3399    let n = jobs.len();
3400    let originals: Vec<SeatJob> = jobs;
3401    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
3402    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
3403    let mut pending: Vec<usize> = (0..n).collect();
3404
3405    for attempt in 0..=retries {
3406        if pending.is_empty() {
3407            break;
3408        }
3409        let mut batch = Vec::with_capacity(pending.len());
3410        for &i in &pending {
3411            let src = &originals[i];
3412            // The prompt and the budget are one decision: a nudge restates
3413            // finished work, a re-sent prompt redoes it.
3414            let (prompt, timeout) = if attempt == 0 {
3415                (src.prompt.clone(), src.timeout)
3416            } else {
3417                let why = done[i]
3418                    .as_ref()
3419                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
3420                    .unwrap_or_else(|| "no parsable answer".to_owned());
3421                let nudge = prompt::nudge(&why);
3422                let nudged = has_context(&src.spec, &seats[i], src.sessions);
3423                let prompt = if nudged {
3424                    nudge
3425                } else {
3426                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
3427                };
3428                (prompt, retry_budget(src.timeout, nudged))
3429            };
3430            batch.push(SeatJob {
3431                spec: src.spec.clone(),
3432                seat: seats[i].clone(),
3433                cwd: src.cwd.clone(),
3434                prompt,
3435                timeout,
3436                allow_write: src.allow_write,
3437                sessions: src.sessions,
3438                artifacts: src.artifacts.clone(),
3439                stem: if attempt == 0 {
3440                    src.stem.clone()
3441                } else {
3442                    format!("{}-retry{attempt}", src.stem)
3443                },
3444            });
3445        }
3446
3447        if attempt > 0 {
3448            let seats_out: Vec<&str> = pending
3449                .iter()
3450                .map(|&i| originals[i].seat.key.as_str())
3451                .collect();
3452            state.event(
3453                ctx.node,
3454                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3455            );
3456        }
3457        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3458        let mut still = Vec::new();
3459        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3460            seats[i] = seat;
3461            let (parsed, quota) = match out {
3462                AgentOutcome::Ok(o) => (
3463                    match verdict::extract_json::<T>(&o.text) {
3464                        Ok(v) => match validate(&v) {
3465                            Ok(()) => Ok((v, o)),
3466                            Err(e) => Err(e),
3467                        },
3468                        Err(e) => Err(e),
3469                    },
3470                    false,
3471                ),
3472                AgentOutcome::Quota(o) => {
3473                    losses.push(QuotaLoss {
3474                        seat: originals[i].seat.key.clone(),
3475                        node: ctx.node.to_owned(),
3476                        at: Timestamp::now(),
3477                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3478                    });
3479                    (
3480                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
3481                        true,
3482                    )
3483                }
3484                // Not a parseable answer, but also not worth a special-cased
3485                // retry here: the nudge loop above already re-asks anything
3486                // that fails to parse, which is exactly what a dropped stream
3487                // needs. Just don't hand its raw error JSON to `extract_json`.
3488                AgentOutcome::Dropped(o) => {
3489                    let why = o
3490                        .dropped
3491                        .as_ref()
3492                        .map(|d| d.why.as_str())
3493                        .unwrap_or("the CLI ended the stream without delivering its answer");
3494                    (
3495                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
3496                        false,
3497                    )
3498                }
3499                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
3500            };
3501            let failed = parsed.is_err();
3502            done[i] = Some(parsed);
3503            // Do not re-ask a rate-limited seat (quota) — a retry is known to
3504            // fail the same way; and never re-ask a seat that already parsed.
3505            if failed && !quota {
3506                still.push(i);
3507            }
3508        }
3509        pending = still;
3510    }
3511
3512    seats
3513        .into_iter()
3514        .zip(done)
3515        .map(|(seat, res)| {
3516            (
3517                seat,
3518                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
3519            )
3520        })
3521        .collect()
3522}
3523
3524/// Describe one verify command's outcome for the event log, distinguishing a
3525/// build/link failure — the toolchain never produced a binary to run — from
3526/// an actual test failure, since only the latter is a verdict on the patch.
3527fn e2e_outcome_label(o: &CommandOutcome) -> String {
3528    if o.ok() {
3529        return "pass".to_owned();
3530    }
3531    let reason = if o.build_failed() {
3532        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
3533    } else {
3534        format!("FAIL ({:?})", o.code)
3535    };
3536    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
3537}
3538
3539/// Run configured shell commands in `cwd`, in order.
3540async fn run_commands(
3541    shell: &[String],
3542    commands: &[String],
3543    cwd: &Path,
3544    timeout: Duration,
3545) -> Vec<CommandOutcome> {
3546    let mut out = Vec::new();
3547    for command in commands {
3548        let started = Instant::now();
3549        let mut cmd = tokio::process::Command::new(&shell[0]);
3550        cmd.quiet();
3551        cmd.args(&shell[1..])
3552            .arg(command)
3553            .current_dir(cwd)
3554            .stdin(std::process::Stdio::null())
3555            .stdout(std::process::Stdio::piped())
3556            .stderr(std::process::Stdio::piped())
3557            .kill_on_drop(true);
3558        let spawned = cmd.spawn();
3559        let (code, body) = match spawned {
3560            Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
3561                Ok(Ok(o)) => {
3562                    let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
3563                    body.push_str(&String::from_utf8_lossy(&o.stderr));
3564                    (o.status.code(), body)
3565                }
3566                Ok(Err(e)) => (None, format!("failed to run: {e}")),
3567                Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
3568            },
3569            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
3570        };
3571        out.push(CommandOutcome {
3572            command: command.clone(),
3573            code,
3574            output_tail: tail(&body, OUTPUT_TAIL),
3575            duration_ms: started.elapsed().as_millis() as u64,
3576        });
3577    }
3578    out
3579}
3580
3581/// The merge commit / pull request body: the task, and — when the winning
3582/// review round was not clean — the findings still open and whatever the
3583/// fixer declined, so `merge = "pr"` hands the reader the same material
3584/// `magi show` does rather than a pull request that reads clean while
3585/// `run.json` disagrees.
3586fn pr_body(state: &RunState, winner: char) -> String {
3587    let mut message = format!(
3588        "Merge magi run {} (candidate {winner})\n\n{}",
3589        state.id, state.instruction
3590    );
3591
3592    let open = state.open_findings();
3593    if !open.is_empty() {
3594        message.push_str("\n\n## Open review findings\n\n");
3595        for f in &open {
3596            message.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
3597        }
3598    }
3599
3600    if let Some(fix) = state.reviews.last().and_then(|r| r.fix.as_ref())
3601        && !fix.rejected.is_empty()
3602    {
3603        message.push_str("\n## Declined by the fixer\n\n");
3604        for r in &fix.rejected {
3605            message.push_str(&format!("- `{}`: {}\n", r.id, r.why));
3606        }
3607    }
3608
3609    message
3610}
3611
3612/// `gh pr create`, returning the PR url.
3613async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
3614    let title = body.lines().next().unwrap_or("magi run").to_owned();
3615    let out = tokio::process::Command::new("gh")
3616        .args([
3617            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
3618        ])
3619        .current_dir(cwd)
3620        .stdin(std::process::Stdio::null())
3621        .output()
3622        .await
3623        .context("spawn gh")?;
3624    if out.status.success() {
3625        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
3626    } else {
3627        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
3628    }
3629}
3630
3631/// Tear a run's worktrees and branches down.
3632pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
3633    let repo = state.repo.clone();
3634    let root = state.worktree_root();
3635    let winner = state.tally.as_ref().map(|t| t.winner);
3636    let mut removed = Vec::new();
3637
3638    for i in 0..state.candidates.len() {
3639        let c = state.candidates[i].clone();
3640        let is_winner = Some(c.label) == winner;
3641        if is_winner && !drop_winner {
3642            continue;
3643        }
3644        if c.worktree.exists() {
3645            git::worktree_remove(&repo, &c.worktree).await.ok();
3646            removed.push(c.worktree.to_string_lossy().into_owned());
3647        }
3648        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
3649            git::branch_delete(&repo, &c.branch).await.ok();
3650            removed.push(c.branch.clone());
3651        }
3652        state.candidates[i].folded = true;
3653    }
3654
3655    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
3656        let path = name.path();
3657        let keep = !drop_winner
3658            && winner.is_some_and(|w| {
3659                path.file_name()
3660                    .is_some_and(|n| n == format!("cand-{w}").as_str())
3661            });
3662        if keep {
3663            continue;
3664        }
3665        git::worktree_remove(&repo, &path).await.ok();
3666        removed.push(path.to_string_lossy().into_owned());
3667    }
3668
3669    if state.enabled_worktree_config && drop_winner {
3670        // A release, not a raw disable: some sibling run in this repository
3671        // may still hold its own reference (see `git::acquire_worktree_config`),
3672        // and only the last release actually turns the setting back off.
3673        git::release_worktree_config(&repo).await.ok();
3674        state.enabled_worktree_config = false;
3675    }
3676    state.save()?;
3677    Ok(removed)
3678}
3679
3680/// Severity of the worst open finding in the last review round, for reporting.
3681pub fn worst_open(state: &RunState) -> Option<Severity> {
3682    state
3683        .reviews
3684        .last()?
3685        .reviews
3686        .iter()
3687        .flat_map(|r| r.findings.iter())
3688        .map(|f| f.severity)
3689        .max()
3690}
3691
3692#[cfg(test)]
3693mod tests {
3694    use super::*;
3695    use std::time::Duration;
3696
3697    // `round_is_clean` is the exact decision this task fixed: a round with a
3698    // seat that never answered must not read the same as a round every seat
3699    // actually reviewed. These are deterministic and process-free by design —
3700    // the equivalent end-to-end check (a real reviewer timing out under a
3701    // live graph run) is a genuine race against wall-clock contention, and a
3702    // spawn slow enough to blow even a generous budget under a loaded test
3703    // run must not turn this specific regression check flaky.
3704
3705    #[test]
3706    fn a_full_panel_that_found_nothing_is_clean() {
3707        assert!(round_is_clean(0, true, 2, 2, IncompleteReviewPolicy::Block));
3708    }
3709
3710    #[test]
3711    fn a_missing_seat_is_never_clean_under_the_default_policy() {
3712        assert!(!round_is_clean(
3713            0,
3714            true,
3715            1,
3716            2,
3717            IncompleteReviewPolicy::Block
3718        ));
3719    }
3720
3721    #[test]
3722    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
3723        assert!(!round_is_clean(1, true, 1, 2, IncompleteReviewPolicy::Warn));
3724    }
3725
3726    #[test]
3727    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
3728        assert!(round_is_clean(0, true, 1, 2, IncompleteReviewPolicy::Warn));
3729    }
3730
3731    #[test]
3732    fn a_full_panel_with_an_open_finding_is_not_clean() {
3733        assert!(!round_is_clean(
3734            1,
3735            true,
3736            2,
3737            2,
3738            IncompleteReviewPolicy::Block
3739        ));
3740    }
3741
3742    #[test]
3743    fn a_full_panel_with_a_red_e2e_is_not_clean() {
3744        assert!(!round_is_clean(
3745            0,
3746            false,
3747            2,
3748            2,
3749            IncompleteReviewPolicy::Block
3750        ));
3751    }
3752
3753    // `review_conclusion` is the exact decision the review hand-off task
3754    // fixed: a round budget spent (or a tree that stopped moving) must not
3755    // collapse into `Blocked` regardless of what verification actually
3756    // said. Deterministic and process-free for the same reason the
3757    // `round_is_clean` family above is.
3758    fn review_round(
3759        clean: bool,
3760        blocking: usize,
3761        answered: usize,
3762        expected: usize,
3763        progressed: bool,
3764        e2e_ok: bool,
3765    ) -> ReviewRound {
3766        ReviewRound {
3767            round: 1,
3768            head: "h".to_owned(),
3769            reviews: Vec::new(),
3770            e2e: vec![CommandOutcome {
3771                command: "test".to_owned(),
3772                code: Some(if e2e_ok { 0 } else { 1 }),
3773                output_tail: String::new(),
3774                duration_ms: 0,
3775            }],
3776            verify_retried: false,
3777            fix: None,
3778            blocking,
3779            answered,
3780            expected,
3781            clean,
3782            progressed,
3783            vote_split: false,
3784            reconsideration: Vec::new(),
3785            verdict: None,
3786        }
3787    }
3788
3789    #[test]
3790    fn review_conclusion_is_none_when_nothing_has_run() {
3791        assert_eq!(review_conclusion(&[], 3), None);
3792    }
3793
3794    #[test]
3795    fn review_conclusion_is_none_while_rounds_remain() {
3796        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
3797        assert_eq!(review_conclusion(&rounds, 3), None);
3798    }
3799
3800    #[test]
3801    fn review_conclusion_is_gating_once_a_round_is_clean() {
3802        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
3803        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
3804    }
3805
3806    #[test]
3807    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
3808        let rounds = vec![
3809            review_round(false, 1, 2, 2, true, true),
3810            review_round(false, 1, 2, 2, true, true),
3811        ];
3812        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
3813    }
3814
3815    #[test]
3816    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
3817        let rounds = vec![
3818            review_round(false, 1, 2, 2, true, true),
3819            review_round(false, 1, 2, 2, true, false),
3820        ];
3821        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
3822    }
3823
3824    #[test]
3825    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
3826        // Missing input, not a verified tree — never a hand-off candidate.
3827        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
3828        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
3829    }
3830
3831    #[test]
3832    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
3833        let rounds = vec![
3834            review_round(false, 1, 2, 2, false, true),
3835            review_round(false, 1, 2, 2, false, true),
3836        ];
3837        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
3838    }
3839
3840    fn secs(n: u64) -> Duration {
3841        Duration::from_secs(n)
3842    }
3843
3844    /// A throwaway repo with one commit on `main`, for tests that need `merge`
3845    /// to make real (and, if it runs at all, real*ly fail*) git calls.
3846    fn init_repo(dir: &Path) {
3847        let run = |args: &[&str]| {
3848            let out = std::process::Command::new("git")
3849                .args(args)
3850                .current_dir(dir)
3851                .output()
3852                .expect("spawn git");
3853            assert!(
3854                out.status.success(),
3855                "git {args:?} failed: {}",
3856                String::from_utf8_lossy(&out.stderr)
3857            );
3858        };
3859        run(&["init", "-b", "main"]);
3860        run(&["config", "user.name", "magi test"]);
3861        run(&["config", "user.email", "magi@example.com"]);
3862        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
3863        run(&["add", "-A"]);
3864        run(&["commit", "-m", "init"]);
3865    }
3866
3867    /// `status == Ready` used to be read as "this is the harmless
3868    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
3869    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
3870    /// run's PR was closed without merging — and reentering `merge` with
3871    /// `mode` still `Pr` does not know the difference, so it pushed and
3872    /// opened a second pull request. `mode == Local` reproduces the same
3873    /// blind spot without a network call: reentry must not attempt another
3874    /// git merge once this node has already recorded an outcome.
3875    #[tokio::test]
3876    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
3877        let tmp = tempfile::tempdir().expect("tempdir");
3878        let repo = tmp.path().join("repo");
3879        std::fs::create_dir_all(&repo).unwrap();
3880        init_repo(&repo);
3881
3882        let mut config = Config::default();
3883        config.merge.mode = MergeMode::Local;
3884
3885        let mut state = RunState::new(
3886            repo.clone(),
3887            "main".to_owned(),
3888            "deadbeef".to_owned(),
3889            "task".to_owned(),
3890            config,
3891        );
3892        state.candidates = vec![Candidate {
3893            index: 0,
3894            label: 'A',
3895            agent: "alpha".to_owned(),
3896            branch: "does-not-exist".to_owned(),
3897            worktree: repo.clone(),
3898            summary: String::new(),
3899            stat: String::new(),
3900            files: 0,
3901            commits: 0,
3902            empty: false,
3903            failed: None,
3904            duration_ms: 0,
3905            folded: false,
3906        }];
3907        state.tally = Some(Tally {
3908            first_choice: BTreeMap::from([('A', 1)]),
3909            borda: BTreeMap::new(),
3910            winner: 'A',
3911            rankings: 1,
3912            unanimous_initial: true,
3913            deliberated: false,
3914            changed_votes: 0,
3915            unanimous_final: true,
3916            tie_break: None,
3917            judges: 0,
3918            present: 0,
3919            quorum: 0,
3920            met_quorum: true,
3921            uncontested: Some("only candidate A produced a change".to_owned()),
3922        });
3923        state.reviews = vec![ReviewRound {
3924            round: 1,
3925            head: "deadbeef".to_owned(),
3926            reviews: Vec::new(),
3927            e2e: Vec::new(),
3928            fix: None,
3929            blocking: 0,
3930            answered: 0,
3931            expected: 0,
3932            clean: true,
3933            verify_retried: false,
3934            progressed: false,
3935            vote_split: false,
3936            reconsideration: Vec::new(),
3937            verdict: None,
3938        }];
3939        state.gate = vec![CommandOutcome {
3940            command: "test".to_owned(),
3941            code: Some(0),
3942            output_tail: String::new(),
3943            duration_ms: 0,
3944        }];
3945        // Reached its conclusion already — e.g. `land` closing the PR without
3946        // merging it, which (like the honest `MergeMode::None` path) leaves
3947        // `status` at `Ready`. The recorded outcome is what actually marks
3948        // this node done.
3949        state.status = RunStatus::Ready;
3950        state.merge = Some(MergeOutcome {
3951            mode: MergeMode::Local,
3952            ok: false,
3953            detail: "already concluded".to_owned(),
3954        });
3955
3956        let mut runner = Runner {
3957            state,
3958            roles: ResolvedRoles {
3959                implementers: Vec::new(),
3960                judges: Vec::new(),
3961                reviewers: Vec::new(),
3962                fixer: None,
3963            },
3964            sem: Arc::new(Semaphore::new(1)),
3965            pause: Pause::new(),
3966        };
3967
3968        runner.merge().await.expect("merge");
3969
3970        assert_eq!(
3971            runner.state.status,
3972            RunStatus::Ready,
3973            "a concluded run's status must not change on reentry"
3974        );
3975        assert_eq!(
3976            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
3977            Some("already concluded"),
3978            "merge must not run again once the node already recorded an outcome"
3979        );
3980    }
3981
3982    #[tokio::test]
3983    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
3984        crate::run::set_home(std::env::temp_dir().join("magi-graph-test-home"));
3985        let tmp = tempfile::tempdir().expect("tempdir");
3986        let repo = tmp.path().join("repo");
3987        std::fs::create_dir_all(&repo).unwrap();
3988        init_repo(&repo);
3989
3990        let mut config = Config::default();
3991        config.merge.mode = MergeMode::Pr;
3992        config.graph.land = true;
3993        config.graph.land_approval = false;
3994
3995        let mut state = RunState::new(
3996            repo.clone(),
3997            "main".to_owned(),
3998            "deadbeef".to_owned(),
3999            "task".to_owned(),
4000            config,
4001        );
4002        state.candidates = vec![Candidate {
4003            index: 0,
4004            label: 'A',
4005            agent: "alpha".to_owned(),
4006            branch: "does-not-exist".to_owned(),
4007            worktree: repo.clone(),
4008            summary: String::new(),
4009            stat: String::new(),
4010            files: 0,
4011            commits: 0,
4012            empty: false,
4013            failed: None,
4014            duration_ms: 0,
4015            folded: false,
4016        }];
4017        state.tally = Some(Tally {
4018            first_choice: BTreeMap::from([('A', 1)]),
4019            borda: BTreeMap::new(),
4020            winner: 'A',
4021            rankings: 1,
4022            unanimous_initial: true,
4023            deliberated: false,
4024            changed_votes: 0,
4025            unanimous_final: true,
4026            tie_break: None,
4027            judges: 0,
4028            present: 0,
4029            quorum: 0,
4030            met_quorum: true,
4031            uncontested: Some("only candidate A produced a change".to_owned()),
4032        });
4033        state.reviews = vec![ReviewRound {
4034            round: 1,
4035            head: "deadbeef".to_owned(),
4036            reviews: Vec::new(),
4037            e2e: Vec::new(),
4038            fix: None,
4039            blocking: 0,
4040            answered: 0,
4041            expected: 0,
4042            clean: true,
4043            verify_retried: false,
4044            progressed: false,
4045            vote_split: false,
4046            reconsideration: Vec::new(),
4047            verdict: None,
4048        }];
4049        state.gate = vec![CommandOutcome {
4050            command: "test".to_owned(),
4051            code: Some(0),
4052            output_tail: String::new(),
4053            duration_ms: 0,
4054        }];
4055        // A first pass through `merge` already pushed and opened this pull
4056        // request; `status` is `Landing` because a previous call into `land`
4057        // parked or was interrupted before it reached a terminal outcome.
4058        state.status = RunStatus::Landing;
4059        state.merge = Some(MergeOutcome {
4060            mode: MergeMode::Pr,
4061            ok: true,
4062            detail: "https://example.invalid/x/y/pull/1".to_owned(),
4063        });
4064
4065        let mut runner = Runner {
4066            state,
4067            roles: ResolvedRoles {
4068                implementers: Vec::new(),
4069                judges: Vec::new(),
4070                reviewers: Vec::new(),
4071                fixer: None,
4072            },
4073            sem: Arc::new(Semaphore::new(1)),
4074            pause: Pause::new(),
4075        };
4076
4077        // `execute`, not `merge` directly: the Landing-resume shortcut lives
4078        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
4079        // exactly because `review_loop` would otherwise clobber the marker
4080        // first.
4081        runner.execute().await.expect("execute");
4082
4083        assert_eq!(
4084            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
4085            Some("https://example.invalid/x/y/pull/1"),
4086            "reentry must not push again or open a second pull request over the \
4087             one `land` is already watching"
4088        );
4089        assert_ne!(
4090            runner.state.status,
4091            RunStatus::Landing,
4092            "land could not actually reach the fake pull request, so it must \
4093             have given up rather than left the run silently parked forever"
4094        );
4095    }
4096
4097    fn state_with_round(round: ReviewRound) -> RunState {
4098        let mut s = RunState::new(
4099            PathBuf::from("/repo"),
4100            "main".to_owned(),
4101            "abc1234".to_owned(),
4102            "add retries".to_owned(),
4103            Config::default(),
4104        );
4105        s.reviews = vec![round];
4106        s
4107    }
4108
4109    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
4110        crate::verdict::Finding {
4111            id: id.to_owned(),
4112            severity,
4113            file: None,
4114            line: None,
4115            title: title.to_owned(),
4116            detail: String::new(),
4117        }
4118    }
4119
4120    #[test]
4121    fn pr_body_names_open_findings_and_declined_ones() {
4122        let round = ReviewRound {
4123            round: 2,
4124            head: "deadbee".to_owned(),
4125            reviews: vec![ReviewRecord {
4126                reviewer: 1,
4127                agent: "alpha".to_owned(),
4128                summary: String::new(),
4129                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
4130                vote: None,
4131                failed: None,
4132                duration_ms: 0,
4133            }],
4134            e2e: vec![CommandOutcome {
4135                command: "cargo test".to_owned(),
4136                code: Some(0),
4137                output_tail: String::new(),
4138                duration_ms: 0,
4139            }],
4140            verify_retried: false,
4141            fix: Some(FixRecord {
4142                agent: "alpha".to_owned(),
4143                addressed: Vec::new(),
4144                rejected: vec![crate::verdict::Rejection {
4145                    id: "R1-1-1".to_owned(),
4146                    why: "not reachable from any caller".to_owned(),
4147                }],
4148                notes: String::new(),
4149                committed: true,
4150                failed: None,
4151                duration_ms: 0,
4152            }),
4153            blocking: 0,
4154            answered: 1,
4155            expected: 1,
4156            clean: false,
4157            progressed: true,
4158            vote_split: false,
4159            reconsideration: Vec::new(),
4160            verdict: None,
4161        };
4162        let state = state_with_round(round);
4163        let body = pr_body(&state, 'A');
4164
4165        assert!(body.contains("add retries"), "the task must still be there");
4166        assert!(body.contains("R2-1-1"), "{body}");
4167        assert!(body.contains("unused import"), "{body}");
4168        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
4169        assert!(
4170            body.contains("not reachable from any caller"),
4171            "the reason it was declined: {body}"
4172        );
4173    }
4174
4175    #[test]
4176    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
4177        let round = ReviewRound {
4178            round: 1,
4179            head: "deadbee".to_owned(),
4180            reviews: vec![ReviewRecord {
4181                reviewer: 1,
4182                agent: "alpha".to_owned(),
4183                summary: String::new(),
4184                findings: Vec::new(),
4185                vote: None,
4186                failed: None,
4187                duration_ms: 0,
4188            }],
4189            e2e: Vec::new(),
4190            verify_retried: false,
4191            fix: None,
4192            blocking: 0,
4193            answered: 1,
4194            expected: 1,
4195            clean: true,
4196            progressed: false,
4197            vote_split: false,
4198            reconsideration: Vec::new(),
4199            verdict: None,
4200        };
4201        let state = state_with_round(round);
4202        let body = pr_body(&state, 'A');
4203        assert!(!body.contains("Open review findings"), "{body}");
4204        assert!(!body.contains("Declined"), "{body}");
4205    }
4206
4207    #[test]
4208    fn a_nudge_gets_a_quarter_of_the_budget() {
4209        // The judge and implement budgets magi ships with.
4210        assert_eq!(retry_budget(secs(1200), true), secs(300));
4211        assert_eq!(retry_budget(secs(3600), true), secs(900));
4212    }
4213
4214    #[test]
4215    fn a_resent_prompt_keeps_the_whole_budget() {
4216        // The seat kept no context, so the retry is the original job again and
4217        // shortening it would only guarantee a second failure.
4218        assert_eq!(retry_budget(secs(1200), false), secs(1200));
4219        assert_eq!(retry_budget(secs(60), false), secs(60));
4220    }
4221
4222    #[test]
4223    fn the_floor_never_exceeds_the_original_budget() {
4224        // A short configured timeout must not be *raised* by the floor: the
4225        // operator asked for a bound, and a retry may not outlast the attempt
4226        // it is retrying.
4227        assert_eq!(retry_budget(secs(60), true), secs(60));
4228        assert_eq!(retry_budget(secs(480), true), secs(120));
4229        assert_eq!(retry_budget(secs(0), true), secs(0));
4230    }
4231}