Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::agent::{self, AgentOutput, Invocation, SeatState};
30use crate::blind;
31use crate::config::{
32    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, Prompts, ResolvedRoles,
33};
34use crate::git;
35use crate::land;
36use crate::proc::Quiet as _;
37use crate::prompt::{self, CandidateView, Turn};
38use crate::run::{
39    BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
40    MergeOutcome, QuotaLoss, ReviewRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord,
41    tail, write_artifact,
42};
43use crate::verdict::{self, FinalVote, FixReport, Position, Ranking, Review, Severity};
44
45/// How much verification output is kept and fed back to the fixer.
46const OUTPUT_TAIL: usize = 8_000;
47
48/// Bytes of a failing command's output kept in an event, so the reason a run
49/// stopped is readable from the report without opening `run.json`.
50const EVENT_OUTPUT_TAIL: usize = 2_000;
51
52/// Consecutive review rounds with no tree progress (see
53/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
54/// instead of spending the rest of the round budget.
55///
56/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
57/// legitimately finds nothing left to change (its previous round's fix already
58/// covered it, and this round's reviewers re-raised only nits) looks the same
59/// as one that is spinning, for exactly one round. Two in a row is where the
60/// two stop being distinguishable, and a review round on this workload has
61/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
62/// third attempt at a tree that has not moved twice running is pure cost.
63/// This does not touch `review_rounds` itself, which stays the operator's
64/// call.
65pub(crate) const STAGNANT_LIMIT: usize = 2;
66
67/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
68/// a base that moved before giving up and leaving the run `Blocked` for a
69/// person.
70///
71/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
72/// that keeps moving faster than a run can catch it is not something more
73/// rebasing fixes, it is a person's call. Not the same *number as*
74/// `land_rounds` - this budget is spent before a pull request exists, land's
75/// after - but bounded for the identical reason, so it uses the same
76/// default. Counted across both call sites in [`Runner::finish_after_tally`]
77/// (once before review, once before the gate), because either one finding
78/// the base still moving is the same signal.
79const BASE_SYNC_ROUNDS: usize = 4;
80
81/// One queued agent invocation.
82///
83/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
84/// CLI hung up on its own stream is asked again from the same job rather than
85/// rebuilt from scratch. See [`Runner::resume_undelivered`].
86#[derive(Clone)]
87struct SeatJob {
88    spec: AgentSpec,
89    seat: SeatState,
90    cwd: PathBuf,
91    prompt: String,
92    timeout: Duration,
93    allow_write: bool,
94    sessions: bool,
95    artifacts: PathBuf,
96    stem: String,
97}
98
99/// How the graph reads one agent invocation.
100///
101/// Quota is split out from an ordinary failure on purpose: a rate-limited call
102/// is known to fail again if retried now, so the retry loop must not spend an
103/// attempt on it. `Dropped` is split out for the opposite reason: unlike
104/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
105/// error JSON, never the agent's answer — a caller that matched only
106/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
107/// left to read that JSON as if it were usable output. `resume_undelivered`
108/// is the only caller that acts on it; everywhere else it is reported like an
109/// ordinary failure.
110enum AgentOutcome {
111    /// A usable output.
112    Ok(AgentOutput),
113    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
114    Quota(AgentOutput),
115    /// The CLI hung up on its own stream after billed work. See
116    /// [`agent::AgentOutput::work_undelivered`].
117    Dropped(AgentOutput),
118    /// Any other failure: a timeout, a bad exit code, an empty reply.
119    Failed(String),
120}
121
122/// A request to park the run at its next node boundary.
123///
124/// Cloning is how the request travels: the loop keeps one handle and hands a
125/// clone to each [`Runner`], and every clone points at the same flag. There
126/// is no channel because there is nothing to send - the only message is
127/// "park", it is idempotent, and a flag cannot be missed by a receiver that
128/// was not listening yet.
129///
130/// The boundary is what makes this cheap. Every node writes the run's state
131/// before the next one starts, and every node skips what is already recorded:
132/// `prep` returns early once candidates exist, `implement` asks only the seats
133/// with nothing on disk, `judge` returns early once judgements exist. So a
134/// parked run resumes into exactly the node it stopped before, and no agent
135/// work is thrown away. Killing the process mid-node, by contrast, loses
136/// whatever the seats in flight had not yet written - which for an implement
137/// wave is an hour of paid work.
138#[derive(Debug, Clone, Default)]
139pub struct Pause(Arc<AtomicBool>);
140
141impl Pause {
142    /// A pause nobody has asked for yet.
143    #[must_use]
144    pub fn new() -> Self {
145        Self::default()
146    }
147
148    /// Ask the run to park at its next node boundary. Idempotent.
149    pub fn park(&self) {
150        self.0.store(true, Ordering::SeqCst);
151    }
152
153    /// Has a park been asked for?
154    #[must_use]
155    pub fn parked(&self) -> bool {
156        self.0.load(Ordering::SeqCst)
157    }
158}
159
160/// Drives one run.
161pub struct Runner {
162    /// Run state; public so the CLI can report on it.
163    pub state: RunState,
164    roles: ResolvedRoles,
165    sem: Arc<Semaphore>,
166    /// Set when someone wants the run parked at its next node boundary.
167    pause: Pause,
168}
169
170/// The commit a run branches from: the base branch as the remote has it.
171///
172/// Two failures this replaces. A run used to branch off `HEAD` and so refused
173/// to start on a dirty tree, which made `magi serve` decline every task for as
174/// long as the operator had work in progress - most of the time. Branching off
175/// the *local* base branch fixed that and introduced a worse one: `land` merges
176/// the winner on GitHub, nothing updates the local ref, and the next run
177/// branches off a base missing everything the previous runs landed. Two tasks
178/// in a row from a phone would have had the second silently re-implementing
179/// against stale code and opening a pull request that reverted the first.
180///
181/// Only refs move here - no checkout, no local branch, no merge - so it is safe
182/// with uncommitted work in the tree. A machine with no network still starts:
183/// the fetch may fail and the local tip is used with a warning, because
184/// refusing to run offline is a worse failure than running against a base the
185/// operator can see for themselves.
186///
187/// One function, called by both entry points. Two answers to "where does a run
188/// branch from" is the kind of drift nobody notices until a diff is wrong.
189async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
190    let tracking = format!("{remote}/{base_branch}");
191    let fetched = git::fetch(repo, remote, base_branch).await;
192    if let Ok(out) = &fetched
193        && out.ok()
194        && git::rev_exists(repo, &tracking).await
195    {
196        return git::rev_parse(repo, &tracking).await;
197    }
198    let why = match &fetched {
199        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
200        Ok(_) => format!("{remote} has no {base_branch}"),
201        Err(e) => e.to_string(),
202    };
203    tracing::warn!(
204        "could not read {tracking} ({why}); branching off the local \
205         {base_branch} instead, which may be behind"
206    );
207    git::rev_parse(repo, base_branch).await.with_context(|| {
208        format!(
209            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
210             branch that exists"
211        )
212    })
213}
214
215impl Runner {
216    /// Start a fresh run against `repo`.
217    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
218        let repo = git::toplevel(repo).await?;
219        let missing = agent::missing_programs(&config.agents);
220        if !missing.is_empty() {
221            bail!(
222                "these agent programs are not on PATH: {}. Fix the roster in \
223                 magi.toml or install them.",
224                missing.join(", ")
225            );
226        }
227        let base_branch = match config.merge.base.clone() {
228            Some(b) => b,
229            None => git::current_branch(&repo)
230                .await?
231                .context("HEAD is detached; set [merge] base in magi.toml")?,
232        };
233        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
234        // Still worth saying out loud. The operator's uncommitted work is not
235        // part of this run, and someone watching a candidate fail to use a
236        // change they just made deserves to know why.
237        if !git::is_clean(&repo).await? {
238            tracing::warn!(
239                "{} has uncommitted changes; they are not part of this run, \
240                 which branches off {base_branch} ({})",
241                repo.display(),
242                &base_commit[..base_commit.len().min(8)]
243            );
244        }
245        let roles = config.resolve_roles()?;
246        let max_parallel = config.graph.max_parallel.max(1);
247        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
248        state.event("start", format!("run {} created", state.id));
249        state.save()?;
250        Ok(Self {
251            state,
252            roles,
253            sem: Arc::new(Semaphore::new(max_parallel)),
254            pause: Pause::new(),
255        })
256    }
257
258    /// Open a review-only run against work that already exists on `branch`.
259    ///
260    /// The expensive half of the graph is the implement wave — measured at
261    /// 111 and 134 internal tool-loop turns on this repository, against a
262    /// handful for a judge or a reviewer. The cheap half is worth running on
263    /// hand-written work too, and there was no way to reach it.
264    ///
265    /// No new state and no schema change are needed: a run with **one** viable
266    /// candidate and a tally already decided degrades `execute` to exactly
267    /// review → gate → merge, because `judge` skips a single-candidate field,
268    /// `deliberate` has fewer than two first choices to reconcile, `vote`
269    /// returns early, `tally` is already present and `fold_losers` has no
270    /// losers. Resuming such a run therefore does the right thing as well.
271    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
272        let repo = git::toplevel(repo).await?;
273        let missing = agent::missing_programs(&config.agents);
274        if !missing.is_empty() {
275            bail!(
276                "these agent programs are not on PATH: {}. Fix the roster in \
277                 magi.toml or install them.",
278                missing.join(", ")
279            );
280        }
281        if !git::branch_exists(&repo, branch).await? {
282            bail!("no branch `{branch}` in {}", repo.display());
283        }
284        let base_branch = match config.merge.base.clone() {
285            Some(b) => b,
286            None => git::current_branch(&repo)
287                .await?
288                .context("HEAD is detached; set [merge] base in magi.toml")?,
289        };
290        if base_branch == branch {
291            bail!("`{branch}` is the base branch; there is nothing to review against");
292        }
293        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
294
295        let roles = config.resolve_roles()?;
296        let max_parallel = config.graph.max_parallel.max(1);
297        // The commit subjects are the closest thing to a task statement that
298        // existing work carries, and the reviewers are told as much.
299        let log = git::log_oneline(&repo, &base_commit, branch)
300            .await
301            .unwrap_or_default();
302        let instruction = format!(
303            "Review the work already on branch `{branch}`. There is no task \
304             statement: what the change claims to do is whatever its commits \
305             say.\n\n{}",
306            if log.trim().is_empty() {
307                "(no commit messages)"
308            } else {
309                log.trim()
310            }
311        );
312        let mut state = RunState::new(
313            repo.clone(),
314            base_branch,
315            base_commit.clone(),
316            instruction,
317            config,
318        );
319
320        // An attached worktree, so the fixer's commits land on the branch under
321        // review rather than on a detached head nobody will look at again.
322        let worktree = state.worktree_root().join("under-review");
323        if let Some(parent) = worktree.parent() {
324            tokio::fs::create_dir_all(parent).await.ok();
325        }
326        let path = worktree.to_string_lossy().to_string();
327        git::git(&repo, &["worktree", "add", &path, branch])
328            .await
329            .with_context(|| {
330                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
331            })?;
332
333        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
334            .await
335            .unwrap_or(0);
336        if commits == 0 {
337            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
338        }
339        let files = git::changed_files(&worktree, &base_commit, "HEAD")
340            .await
341            .map(|f| f.len())
342            .unwrap_or(0);
343        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
344            .await
345            .unwrap_or_default();
346
347        state.candidates.push(Candidate {
348            index: 0,
349            label: 'A',
350            // Not an agent id on purpose: nothing in the roster wrote this, and
351            // the stats tables must not credit anyone with a win for it.
352            agent: "(existing branch)".to_owned(),
353            branch: branch.to_owned(),
354            worktree,
355            summary: String::new(),
356            stat,
357            files,
358            commits,
359            empty: false,
360            failed: None,
361            duration_ms: 0,
362            folded: false,
363        });
364        state.tally = Some(Tally {
365            first_choice: BTreeMap::from([('A', 0)]),
366            borda: BTreeMap::new(),
367            winner: 'A',
368            rankings: 0,
369            unanimous_initial: false,
370            deliberated: false,
371            changed_votes: 0,
372            unanimous_final: false,
373            tie_break: None,
374            // No panel sat, so no quorum applies. Zero judges is the correct
375            // number for work that never competed, and must not be reported as
376            // a collapsed panel.
377            judges: 0,
378            present: 0,
379            quorum: 0,
380            met_quorum: true,
381            uncontested: Some("review-only run: nothing competed".to_owned()),
382        });
383        state.status = RunStatus::Reviewing;
384        state.event(
385            "start",
386            format!(
387                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
388                state.id
389            ),
390        );
391        state.save()?;
392        Ok(Self {
393            state,
394            roles,
395            sem: Arc::new(Semaphore::new(max_parallel)),
396            pause: Pause::new(),
397        })
398    }
399
400    /// Reopen an existing run.
401    pub fn resume(id: &str) -> Result<Self> {
402        let state = RunState::load(id)?;
403        let roles = state.config.resolve_roles()?;
404        let max_parallel = state.config.graph.max_parallel.max(1);
405        Ok(Self {
406            state,
407            roles,
408            sem: Arc::new(Semaphore::new(max_parallel)),
409            pause: Pause::new(),
410        })
411    }
412
413    /// Walk the graph to a terminal state, skipping nodes already recorded.
414    pub async fn execute(&mut self) -> Result<()> {
415        // Moving again, so it is no longer parked. Set before the walk rather
416        // than in `resume`, so every way of re-entering the graph clears it
417        // and a card cannot claim a run is waiting to be resumed while the
418        // agents are already working.
419        self.state.parked = false;
420        // Any seat this state still lists as answering belongs to whatever
421        // process last drove this run — this one included, if it crashed
422        // mid-wave. Cleared and flushed immediately, before anything else
423        // runs, so a resume can never show a seat as live when nothing is
424        // asking it anything yet; the node that actually dispatches the next
425        // wave repopulates it.
426        if self.state.clear_active() {
427            self.state.save()?;
428        }
429        // A run that already lost its quorum never resumes into the verdict
430        // machinery: `deliberate` and `vote` would otherwise clobber the
431        // stalled marker back to Voting and the run would keep going past a
432        // verdict that is no longer trustworthy. Everything already recorded is
433        // kept, so the run stays resumable (or foldable) for a human to pick up.
434        //
435        // On --resume the run gets one chance to repair itself: the seats a
436        // rate limit took out are re-asked. If their quota has since reset and
437        // the quorum is restored, the run picks up and finishes; otherwise it
438        // stays stale and still-resumable for a later retry. If it does not
439        // recover, the returned status stays `Stalled` and nothing was
440        // clobbered (the recovery only mutates entries for the lost seats).
441        if self.state.status == RunStatus::Stalled {
442            if self.recover_stall().await? {
443                self.finish_after_tally().await?;
444            } else {
445                // Still below quorum: persist the marker and stay resumable.
446                self.state.save()?;
447            }
448            return Ok(());
449        }
450        self.prep().await?;
451        if self.park_here()? {
452            return Ok(());
453        }
454        self.implement().await?;
455        if self.park_here()? {
456            return Ok(());
457        }
458        self.judge().await?;
459        if self.park_here()? {
460            return Ok(());
461        }
462        self.deliberate().await?;
463        if self.park_here()? {
464            return Ok(());
465        }
466        self.vote().await?;
467        if self.park_here()? {
468            return Ok(());
469        }
470        self.tally()?;
471        // A verdict that lost its quorum is not trustworthy: do not review,
472        // gate, or merge on it. Everything already done is kept, so the run
473        // stays resumable (or foldable); the human can replace the agent that
474        // ran out of quota and pick it up.
475        if self.state.status == RunStatus::Stalled {
476            // Persist the stalled marker now — the normal end-of-execute save
477            // below is below this early return, and without it a resumed run
478            // would reload a pre-tally status and keep going.
479            self.state.save()?;
480            return Ok(());
481        }
482        self.finish_after_tally().await?;
483        Ok(())
484    }
485
486    /// Park here if asked to, recording it in the run's own timeline.
487    ///
488    /// Returns whether the caller should stop walking the graph. The state is
489    /// saved either way by the node that just finished; this adds the event so
490    /// the operator's card says why a run that is neither finished nor moving
491    /// is sitting where it is.
492    fn park_here(&mut self) -> Result<bool> {
493        if !self.pause.parked() {
494            return Ok(false);
495        }
496        self.state.event(
497            "park",
498            format!(
499                "parked after `{}` — resume to carry on from here",
500                self.state.status.as_str()
501            ),
502        );
503        self.state.parked = true;
504        self.state.save()?;
505        Ok(true)
506    }
507
508    /// Hand the runner a pause to watch.
509    pub fn on_pause(&mut self, pause: Pause) {
510        self.pause = pause;
511    }
512
513    /// The tail of the graph after a trustworthy tally: fold losers, review,
514    /// gate, merge, and persist.
515    async fn finish_after_tally(&mut self) -> Result<()> {
516        self.fold_losers().await?;
517        // Before review starts, and again right before the gate: a run's
518        // review rounds can themselves take long enough for the base to move
519        // a second time, and the gate is the one node whose "green" gets
520        // acted on.
521        self.sync_to_base().await?;
522        self.review_loop().await?;
523        self.sync_to_base().await?;
524        self.gate().await?;
525        self.merge().await?;
526        self.state.save()?;
527        Ok(())
528    }
529
530    // ---------------------------------------------------------------- prep
531
532    async fn prep(&mut self) -> Result<()> {
533        if !self.state.candidates.is_empty() {
534            return Ok(());
535        }
536        self.state.status = RunStatus::Prep;
537        let repo = self.state.repo.clone();
538        let base = self.state.base_commit.clone();
539        let root = self.state.worktree_root();
540        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
541
542        // The hook is the write-time half of the blindness contract; the
543        // presentation filter in `blind` is the half that cannot be bypassed.
544        let hooks_dir = self.state.dir().join("hooks");
545        if self.state.config.blind.commit_msg_hook {
546            std::fs::create_dir_all(&hooks_dir)
547                .with_context(|| format!("create {}", hooks_dir.display()))?;
548            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
549            let path = hooks_dir.join("commit-msg");
550            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
551            make_executable(&path)?;
552            if git::enable_worktree_config(&repo).await? {
553                self.state.enabled_worktree_config = true;
554            }
555        }
556
557        for (index, (spec, label)) in self
558            .roles
559            .implementers
560            .clone()
561            .into_iter()
562            .zip(labels)
563            .enumerate()
564        {
565            let branch = self.state.branch_for(label);
566            let worktree = root.join(format!("cand-{label}"));
567            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
568            if self.state.config.blind.commit_msg_hook {
569                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
570            }
571            git::local_exclude(&worktree, "/.magi/").await?;
572            self.state.candidates.push(Candidate {
573                index,
574                label,
575                agent: spec.id.clone(),
576                branch,
577                worktree,
578                summary: String::new(),
579                stat: String::new(),
580                files: 0,
581                commits: 0,
582                empty: false,
583                failed: None,
584                duration_ms: 0,
585                folded: false,
586            });
587        }
588
589        for j in 1..=self.roles.judges.len() {
590            let wt = root.join(format!("judge-{j}"));
591            if !wt.exists() {
592                git::worktree_add_detached(&repo, &wt, &base).await?;
593            }
594        }
595
596        // A judge cannot tell it is looking at its own patch — the seats keep
597        // separate conversations — but a panel that shares agents with the
598        // field is less independent than it looks, and that is worth saying out
599        // loud once per run rather than leaving it in the config.
600        let authors: Vec<&str> = self
601            .roles
602            .implementers
603            .iter()
604            .map(|a| a.id.as_str())
605            .collect();
606        let overlap: Vec<String> = self
607            .roles
608            .judges
609            .iter()
610            .enumerate()
611            .filter(|(_, j)| authors.contains(&j.id.as_str()))
612            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
613            .collect();
614        if !overlap.is_empty() {
615            let note = format!(
616                "{} also authored a candidate; blind, but the panel is less \
617                 independent than {} distinct agents would be",
618                overlap.join(", "),
619                self.roles.judges.len()
620            );
621            self.state.event("prep", note);
622        }
623
624        self.state.event(
625            "prep",
626            format!(
627                "{} candidates, {} judges, base {} ({})",
628                self.state.candidates.len(),
629                self.roles.judges.len(),
630                &self.state.base_commit[..7.min(self.state.base_commit.len())],
631                self.state.base_branch
632            ),
633        );
634        self.state.status = RunStatus::Implementing;
635        self.state.save()?;
636        Ok(())
637    }
638
639    // ----------------------------------------------------------- implement
640
641    async fn implement(&mut self) -> Result<()> {
642        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
643        // agent files with `magi task add` name the run that paid for it. The
644        // prompt overlay is cloned alongside it because the waves borrow it
645        // while `self` is mutably borrowed by the node's own bookkeeping.
646        let run_id = self.state.id.clone();
647        let prompts = self.state.config.prompts.clone();
648        let todo: Vec<usize> = self
649            .state
650            .candidates
651            .iter()
652            .enumerate()
653            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
654            .map(|(i, _)| i)
655            .collect();
656        if todo.is_empty() {
657            return self.after_implement();
658        }
659        self.state.status = RunStatus::Implementing;
660
661        let language = self.state.config.graph.language.clone();
662        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
663        let sessions = self.state.config.graph.sessions;
664        let artifacts = agent::artifacts_dir(&self.state.dir());
665
666        let mut jobs = Vec::new();
667        for &i in &todo {
668            let (index, label, worktree) = {
669                let c = &self.state.candidates[i];
670                (c.index, c.label, c.worktree.clone())
671            };
672            let spec = self.roles.implementers[index].clone();
673            let seat_key = format!("impl-{label}");
674            let seat = self.seat(&seat_key, &spec.id);
675            let instruction = self.state.instruction.clone();
676            jobs.push(SeatJob {
677                spec,
678                seat,
679                prompt: prompt::implement(&instruction, &worktree.to_string_lossy(), &language),
680                cwd: worktree,
681                timeout,
682                allow_write: true,
683                sessions,
684                artifacts: artifacts.clone(),
685                stem: format!("impl-{label}"),
686            });
687        }
688
689        self.state.event(
690            "implement",
691            format!("{} candidates in parallel", jobs.len()),
692        );
693        // Kept so a seat whose CLI hung up can be asked again from the same
694        // job: `wave` consumes what it is given.
695        let sent = jobs.clone();
696        let cache = self.state.config.cache_dir();
697        let ctx = WaveCtx {
698            run: &run_id,
699            node: "implement",
700            prompts: &prompts,
701            cache: cache.as_deref(),
702        };
703        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
704        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
705            .await;
706
707        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
708            let seat_key = seat.key.clone();
709            self.state.seats.insert(seat.key.clone(), seat);
710            let label = self.state.candidates[i].label;
711            let worktree = self.state.candidates[i].worktree.clone();
712            let base = self.state.base_commit.clone();
713
714            let (summary, duration, failed) = match out {
715                AgentOutcome::Ok(o) => {
716                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
717                    let failed = (!o.usable()).then(|| {
718                        if o.timed_out {
719                            "agent timed out".to_owned()
720                        } else {
721                            format!("agent exited with {:?}", o.exit_code)
722                        }
723                    });
724                    (text, o.duration_ms, failed)
725                }
726                // Left un-resumed by `resume_undelivered` (a dirty tree
727                // already rescues the work, or there was no session left to
728                // resume into) — reported like the ordinary failure it is,
729                // never as if `o.text` (the CLI's raw error JSON) were an
730                // answer.
731                AgentOutcome::Dropped(o) => {
732                    let why = o
733                        .dropped
734                        .as_ref()
735                        .map(|d| d.why.as_str())
736                        .unwrap_or("the CLI ended the stream without delivering its answer");
737                    (
738                        String::new(),
739                        o.duration_ms,
740                        Some(format!("the CLI dropped the stream ({why})")),
741                    )
742                }
743                AgentOutcome::Quota(o) => {
744                    self.state.quota.push(QuotaLoss {
745                        seat: seat_key,
746                        node: "implement".to_owned(),
747                        at: Timestamp::now(),
748                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
749                    });
750                    (
751                        String::new(),
752                        o.duration_ms,
753                        Some("rate limited (quota); produced no change".to_owned()),
754                    )
755                }
756                AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
757            };
758
759            // Rescue anything the agent edited but never committed: an
760            // uncommitted candidate would silently be an empty one.
761            let rescued = git::commit_all(
762                &worktree,
763                &format!("magi: candidate {label} (uncommitted work)"),
764            )
765            .await
766            .unwrap_or(false);
767            let commits = git::commits_ahead(&worktree, &base, "HEAD")
768                .await
769                .unwrap_or(0);
770            let patch = git::diff(&worktree, &base, "HEAD")
771                .await
772                .unwrap_or_default();
773            let stat = git::diff_stat(&worktree, &base, "HEAD")
774                .await
775                .unwrap_or_default();
776            let files = git::changed_files(&worktree, &base, "HEAD")
777                .await
778                .map(|f| f.len())
779                .unwrap_or(0);
780            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
781
782            let c = &mut self.state.candidates[i];
783            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
784            c.stat = stat;
785            c.files = files;
786            c.commits = commits;
787            c.duration_ms = duration;
788            c.empty = commits == 0 || patch.trim().is_empty();
789            // An agent that failed but still produced a committed change stays
790            // in the running: the patch is what gets judged, not the exit code.
791            c.failed = match failed {
792                Some(_) if c.empty => failed,
793                _ => None,
794            };
795            let note = match (&c.failed, c.empty, rescued) {
796                (Some(e), _, _) => format!("candidate {label}: {e}"),
797                (None, true, _) => format!("candidate {label}: no change produced"),
798                (None, false, true) => {
799                    format!(
800                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
801                    )
802                }
803                (None, false, false) => {
804                    format!("candidate {label}: {files} files, {commits} commits")
805                }
806            };
807            self.state.event("implement", note);
808            self.state.save()?;
809        }
810
811        self.after_implement()
812    }
813
814    /// Ask again, once, for work a CLI did and then failed to hand over.
815    ///
816    /// [`agent::dropped_stream`] recognises the one shape observed: an error
817    /// status with an empty response and a usage report showing output tokens,
818    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
819    /// seven minutes and 14,267 output tokens that arrived as an empty
820    /// candidate, because `agy`'s own subscriber fell behind and hung up.
821    ///
822    /// Two conditions, and both matter:
823    ///
824    /// - **Only when the tree is untouched.** Often the agent has already
825    ///   written its files and only the closing message was lost; the rescue
826    ///   commit below picks that up and there is nothing to ask for. Re-asking
827    ///   then would pay for a second implementation of work already on disk.
828    /// - **Once.** A CLI that drops one stream can drop the next, and this
829    ///   node is the most expensive in the graph.
830    ///
831    /// The re-ask is a resume, not a re-run: `has_context` is true because the
832    /// dropped reply still carried its `conversation_id`, so the seat is asked
833    /// to finish what it was doing rather than sent the whole task again. It
834    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
835    /// node's - for the same reason a re-ranked judge does: restating finished
836    /// work is not the work.
837    ///
838    /// Unlike a quota this is worth retrying at all: a rate limit fails the
839    /// same way until it resets, while an abandoned conversation is still
840    /// there to be picked up.
841    async fn resume_undelivered(
842        &mut self,
843        results: &mut [(usize, SeatState, AgentOutcome)],
844        sent: &[SeatJob],
845        prompts: &Prompts,
846        run_id: &str,
847    ) {
848        for (wi, seat, out) in results.iter_mut() {
849            let Some(dropped) = (match &*out {
850                AgentOutcome::Dropped(o) => o.dropped.clone(),
851                _ => None,
852            }) else {
853                continue;
854            };
855            let Some(job) = sent.get(*wi) else { continue };
856            // Already on disk? Then only the closing message was lost.
857            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
858                self.state.event(
859                    "implement",
860                    format!(
861                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
862                         work is in the tree",
863                        seat.key, dropped.output_tokens, dropped.why
864                    ),
865                );
866                continue;
867            }
868            // The re-ask only makes sense as a resume: `resume_after_drop`
869            // says nothing about the task, trusting the seat to still hold it.
870            // Without a session to resume — sessions disabled, or this CLI's
871            // drop shape happened not to carry a session id — that prompt
872            // would open a brand-new conversation with no context at all,
873            // which is worse than leaving this as the ordinary failure it
874            // already is.
875            if !has_context(&job.spec, seat, job.sessions) {
876                self.state.event(
877                    "implement",
878                    format!(
879                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
880                         is no session left to resume",
881                        seat.key, dropped.output_tokens, dropped.why
882                    ),
883                );
884                continue;
885            }
886            self.state.event(
887                "implement",
888                format!(
889                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
890                     conversation",
891                    seat.key, dropped.output_tokens, dropped.why
892                ),
893            );
894            let mut retry = job.clone();
895            retry.seat = seat.clone();
896            retry.prompt = prompt::resume_after_drop(&dropped.why);
897            retry.timeout = retry_budget(job.timeout, true);
898            retry.stem = format!("{}-resume", job.stem);
899            let cache = self.state.config.cache_dir();
900            let ctx = WaveCtx {
901                run: run_id,
902                node: "implement",
903                prompts,
904                cache: cache.as_deref(),
905            };
906            let (resumed_seat, resumed) =
907                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
908            *seat = resumed_seat;
909            *out = resumed;
910        }
911    }
912
913    fn after_implement(&mut self) -> Result<()> {
914        // Scan every candidate patch once the set is complete.
915        if self.state.leaks.is_empty() {
916            let cfg = self.state.config.blind.clone();
917            let mut leaks = Vec::new();
918            for c in &self.state.candidates {
919                let Some(patch) =
920                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
921                else {
922                    continue;
923                };
924                leaks.extend(blind::scan(
925                    &format!("candidate {} patch", c.label),
926                    &patch,
927                    &cfg.vendor_tokens,
928                ));
929            }
930            if !leaks.is_empty() {
931                let summary = leaks
932                    .iter()
933                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
934                    .collect::<Vec<_>>()
935                    .join(", ");
936                match cfg.on_leak {
937                    LeakPolicy::Fail => {
938                        self.state.status = RunStatus::Failed;
939                        self.state
940                            .event("blind", format!("vendor text in a patch: {summary}"));
941                        self.state.leaks = leaks;
942                        self.state.save()?;
943                        bail!(
944                            "blind.on_leak = \"fail\" and vendor text reached a \
945                             judged patch: {summary}"
946                        );
947                    }
948                    LeakPolicy::Redact => self.state.event(
949                        "blind",
950                        format!("redacting vendor text for judging: {summary}"),
951                    ),
952                    LeakPolicy::Warn => self.state.event(
953                        "blind",
954                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
955                    ),
956                }
957                self.state.leaks = leaks;
958            }
959        }
960
961        if self.state.viable().is_empty() {
962            self.state.status = RunStatus::Failed;
963            self.state.save()?;
964            bail!("no candidate produced a change; nothing to judge");
965        }
966        self.state.status = RunStatus::Judging;
967        self.state.save()?;
968        Ok(())
969    }
970
971    // --------------------------------------------------------------- judge
972
973    async fn judge(&mut self) -> Result<()> {
974        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
975        // agent files with `magi task add` name the run that paid for it. The
976        // prompt overlay is cloned alongside it because the waves borrow it
977        // while `self` is mutably borrowed by the node's own bookkeeping.
978        let run_id = self.state.id.clone();
979        let prompts = self.state.config.prompts.clone();
980        if !self.state.judgements.is_empty() || self.state.judge_skipped {
981            return Ok(());
982        }
983        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
984        if viable.len() == 1 {
985            // Recorded so this is a one-time event: `judgements` stays empty
986            // either way, which without this flag is indistinguishable from
987            // "not yet judged" on the next reentry — and status is left
988            // untouched, so a later node's conclusion (e.g. `Blocked` after
989            // the review budget ran out) survives a resume instead of being
990            // clobbered back to `Judging` by this node running again.
991            self.state.judge_skipped = true;
992            self.state.event(
993                "judge",
994                format!(
995                    "only candidate {} produced a change; judging skipped",
996                    viable[0].label
997                ),
998            );
999            self.state.save()?;
1000            return Ok(());
1001        }
1002        self.state.status = RunStatus::Judging;
1003
1004        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1005        let language = self.state.config.graph.language.clone();
1006        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1007        let sessions = self.state.config.graph.sessions;
1008        let artifacts = agent::artifacts_dir(&self.state.dir());
1009        let root = self.state.worktree_root();
1010        let base_short = short(&self.state.base_commit);
1011
1012        let mut jobs = Vec::new();
1013        let mut orders = Vec::new();
1014        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1015            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1016            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1017            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
1018            let seat_key = format!("judge-{}", j + 1);
1019            let seat = self.seat(&seat_key, &spec.id);
1020            jobs.push(SeatJob {
1021                prompt: prompt::judge(
1022                    &self.state.instruction,
1023                    &views,
1024                    self.roles.judges.len(),
1025                    &base_short,
1026                    &language,
1027                ),
1028                spec,
1029                seat,
1030                cwd: root.join(format!("judge-{}", j + 1)),
1031                timeout,
1032                allow_write: false,
1033                sessions,
1034                artifacts: artifacts.clone(),
1035                stem: format!("judge-{}", j + 1),
1036            });
1037        }
1038
1039        self.state.event(
1040            "judge",
1041            format!(
1042                "{} judges ranking {} candidates blind",
1043                jobs.len(),
1044                viable.len()
1045            ),
1046        );
1047        let labels_for_check = labels.clone();
1048        let mut quota_losses = Vec::new();
1049        let cache = self.state.config.cache_dir();
1050        let ctx = WaveCtx {
1051            run: &run_id,
1052            node: "judge",
1053            prompts: &prompts,
1054            cache: cache.as_deref(),
1055        };
1056        let results = ask_json_wave::<Ranking>(
1057            jobs,
1058            Arc::clone(&self.sem),
1059            self.state.config.graph.retries,
1060            &ctx,
1061            &mut quota_losses,
1062            &mut self.state,
1063            &move |r: &Ranking| r.validate(&labels_for_check),
1064        )
1065        .await;
1066        self.state.quota.extend(quota_losses);
1067
1068        for (j, (seat, res)) in results.into_iter().enumerate() {
1069            let agent_id = seat.agent.clone();
1070            self.state.seats.insert(seat.key.clone(), seat);
1071            let mut record = Judgement {
1072                judge: j + 1,
1073                seat: format!("judge-{}", j + 1),
1074                agent: agent_id,
1075                ranking: Vec::new(),
1076                reasons: BTreeMap::new(),
1077                confidence: None,
1078                order: orders[j].clone(),
1079                failed: None,
1080                duration_ms: 0,
1081            };
1082            match res {
1083                Ok((ranking, out)) => {
1084                    record.ranking = ranking.normalized();
1085                    record.reasons = ranking.reasons;
1086                    record.confidence = ranking.confidence;
1087                    record.duration_ms = out.duration_ms;
1088                    self.state.event(
1089                        "judge",
1090                        format!(
1091                            "judge {} ranked {}",
1092                            j + 1,
1093                            record.ranking.iter().collect::<String>()
1094                        ),
1095                    );
1096                }
1097                Err(e) => {
1098                    record.failed = Some(e.to_string());
1099                    self.state
1100                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1101                }
1102            }
1103            self.state.judgements.push(record);
1104            self.state.save()?;
1105        }
1106        Ok(())
1107    }
1108
1109    // ---------------------------------------------------------- deliberate
1110
1111    async fn deliberate(&mut self) -> Result<()> {
1112        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1113        // agent files with `magi task add` name the run that paid for it. The
1114        // prompt overlay is cloned alongside it because the waves borrow it
1115        // while `self` is mutably borrowed by the node's own bookkeeping.
1116        let run_id = self.state.id.clone();
1117        let prompts = self.state.config.prompts.clone();
1118        if !self.state.deliberation.is_empty() {
1119            return Ok(());
1120        }
1121        let tops: Vec<char> = self
1122            .state
1123            .judgements
1124            .iter()
1125            .filter_map(|j| j.ranking.first().copied())
1126            .collect();
1127        let rounds = self.state.config.graph.deliberate_rounds;
1128        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1129            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1130                self.state.event(
1131                    "deliberate",
1132                    format!("judges agreed on {} outright; no deliberation", tops[0]),
1133                );
1134            }
1135            self.state.status = RunStatus::Voting;
1136            self.state.save()?;
1137            return Ok(());
1138        }
1139
1140        self.state.status = RunStatus::Deliberating;
1141        self.state.event(
1142            "deliberate",
1143            format!(
1144                "split: first choices were {} — opening {rounds} round(s)",
1145                tops.iter().collect::<String>()
1146            ),
1147        );
1148
1149        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1150        let language = self.state.config.graph.language.clone();
1151        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1152        let sessions = self.state.config.graph.sessions;
1153        let artifacts = agent::artifacts_dir(&self.state.dir());
1154        let root = self.state.worktree_root();
1155        let base_short = short(&self.state.base_commit);
1156
1157        // Judges argue in sequence so that a turn can answer the one before it;
1158        // that is the difference between deliberation and three parallel
1159        // monologues.
1160        for round in 1..=rounds {
1161            let mut turns: Vec<DeliberationTurn> = Vec::new();
1162            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1163                if self.state.judgements[j].failed.is_some() {
1164                    continue;
1165                }
1166                let seat_key = format!("judge-{}", j + 1);
1167                let mut seat = self.seat(&seat_key, &spec.id);
1168                let transcript = self.transcript(&turns, j);
1169                let context = if has_context(&spec, &seat, sessions) {
1170                    None
1171                } else {
1172                    Some(self.candidate_block(&viable, &base_short))
1173                };
1174                let text = prompt::deliberate(
1175                    &self.state.instruction,
1176                    context.as_deref(),
1177                    &transcript,
1178                    round,
1179                    rounds,
1180                    &language,
1181                );
1182                let job = SeatJob {
1183                    spec,
1184                    seat: seat.clone(),
1185                    prompt: text,
1186                    cwd: root.join(format!("judge-{}", j + 1)),
1187                    timeout,
1188                    allow_write: false,
1189                    sessions,
1190                    artifacts: artifacts.clone(),
1191                    stem: format!("delib-{round}-judge-{}", j + 1),
1192                };
1193                let cache = self.state.config.cache_dir();
1194                let ctx = WaveCtx {
1195                    run: &run_id,
1196                    node: "deliberate",
1197                    prompts: &prompts,
1198                    cache: cache.as_deref(),
1199                };
1200                let (updated, out) =
1201                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1202                seat = updated;
1203                let agent_id = seat.agent.clone();
1204                let seat_key = seat.key.clone();
1205                self.state.seats.insert(seat.key.clone(), seat);
1206                let body = match out {
1207                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1208                    // Never read the CLI's raw error JSON as this judge's
1209                    // position — skip the seat instead, the same as any other
1210                    // failed turn.
1211                    AgentOutcome::Dropped(o) => {
1212                        let why =
1213                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1214                                "the CLI ended the stream without delivering its answer",
1215                            );
1216                        self.state.event(
1217                            "deliberate",
1218                            format!(
1219                                "judge {} skipped: the CLI dropped the stream ({why})",
1220                                j + 1
1221                            ),
1222                        );
1223                        continue;
1224                    }
1225                    AgentOutcome::Quota(o) => {
1226                        self.state.quota.push(QuotaLoss {
1227                            seat: seat_key,
1228                            node: "deliberate".to_owned(),
1229                            at: Timestamp::now(),
1230                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1231                        });
1232                        self.state.event(
1233                            "deliberate",
1234                            format!("judge {} skipped: rate limited (quota)", j + 1),
1235                        );
1236                        continue;
1237                    }
1238                    AgentOutcome::Failed(e) => {
1239                        self.state
1240                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1241                        continue;
1242                    }
1243                };
1244                let tentative = verdict::extract_json::<Position>(&body)
1245                    .ok()
1246                    .and_then(|p| p.tentative)
1247                    .and_then(|s| s.trim().chars().next())
1248                    .map(|c| c.to_ascii_uppercase());
1249                self.state.event(
1250                    "deliberate",
1251                    format!(
1252                        "round {round}: judge {} now favours {}",
1253                        j + 1,
1254                        tentative.map_or("—".to_owned(), |c| c.to_string())
1255                    ),
1256                );
1257                turns.push(DeliberationTurn {
1258                    judge: j + 1,
1259                    agent: agent_id,
1260                    body: blind::sanitize_prose(&body, &self.state.config.blind),
1261                    tentative,
1262                });
1263            }
1264            self.state
1265                .deliberation
1266                .push(DeliberationRound { round, turns });
1267            self.state.save()?;
1268        }
1269
1270        self.state.status = RunStatus::Voting;
1271        self.state.save()?;
1272        Ok(())
1273    }
1274
1275    // ---------------------------------------------------------------- vote
1276
1277    async fn vote(&mut self) -> Result<()> {
1278        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1279        // agent files with `magi task add` name the run that paid for it. The
1280        // prompt overlay is cloned alongside it because the waves borrow it
1281        // while `self` is mutably borrowed by the node's own bookkeeping.
1282        let run_id = self.state.id.clone();
1283        let prompts = self.state.config.prompts.clone();
1284        if !self.state.votes.is_empty() {
1285            return Ok(());
1286        }
1287        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1288        if viable.len() == 1 {
1289            return Ok(());
1290        }
1291        self.state.status = RunStatus::Voting;
1292
1293        let language = self.state.config.graph.language.clone();
1294        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1295        let sessions = self.state.config.graph.sessions;
1296        let artifacts = agent::artifacts_dir(&self.state.dir());
1297        let root = self.state.worktree_root();
1298        let base_short = short(&self.state.base_commit);
1299        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1300
1301        let mut jobs = Vec::new();
1302        let mut seats_at = Vec::new();
1303        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1304            if self
1305                .state
1306                .judgements
1307                .get(j)
1308                .is_some_and(|r| r.failed.is_some())
1309            {
1310                continue;
1311            }
1312            let seat_key = format!("judge-{}", j + 1);
1313            let seat = self.seat(&seat_key, &spec.id);
1314            let mut text = prompt::final_vote(&viable, &language);
1315            if !has_context(&spec, &seat, sessions) {
1316                text = format!(
1317                    "{}\n\n# Candidates\n\n{}",
1318                    text,
1319                    self.candidate_block(&candidates, &base_short)
1320                );
1321            }
1322            jobs.push(SeatJob {
1323                spec,
1324                seat,
1325                prompt: text,
1326                cwd: root.join(format!("judge-{}", j + 1)),
1327                timeout,
1328                allow_write: false,
1329                sessions,
1330                artifacts: artifacts.clone(),
1331                stem: format!("vote-judge-{}", j + 1),
1332            });
1333            seats_at.push(j);
1334        }
1335
1336        self.state.event(
1337            "vote",
1338            format!(
1339                "collecting {} final votes one by one, privately",
1340                jobs.len()
1341            ),
1342        );
1343        let allowed = viable.clone();
1344        let mut quota_losses = Vec::new();
1345        let cache = self.state.config.cache_dir();
1346        let ctx = WaveCtx {
1347            run: &run_id,
1348            node: "vote",
1349            prompts: &prompts,
1350            cache: cache.as_deref(),
1351        };
1352        let results = ask_json_wave::<FinalVote>(
1353            jobs,
1354            Arc::clone(&self.sem),
1355            self.state.config.graph.retries,
1356            &ctx,
1357            &mut quota_losses,
1358            &mut self.state,
1359            &move |v: &FinalVote| match v.label() {
1360                Some(c) if allowed.contains(&c) => Ok(()),
1361                other => bail!("vote {other:?} is not one of {allowed:?}"),
1362            },
1363        )
1364        .await;
1365        self.state.quota.extend(quota_losses);
1366
1367        for (&j, (seat, res)) in seats_at.iter().zip(results) {
1368            let agent_id = seat.agent.clone();
1369            self.state.seats.insert(seat.key.clone(), seat);
1370            let initial = self
1371                .state
1372                .judgements
1373                .get(j)
1374                .and_then(|r| r.ranking.first().copied());
1375            let mut record = VoteRecord {
1376                judge: j + 1,
1377                agent: agent_id,
1378                vote: None,
1379                reason: String::new(),
1380                changed: false,
1381            };
1382            match res {
1383                Ok((v, _)) => {
1384                    record.vote = v.label();
1385                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1386                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1387                    self.state.event(
1388                        "vote",
1389                        format!(
1390                            "judge {} voted {}{}",
1391                            j + 1,
1392                            record.vote.unwrap_or('?'),
1393                            if record.changed { " (changed)" } else { "" }
1394                        ),
1395                    );
1396                }
1397                Err(e) => {
1398                    self.state
1399                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1400                }
1401            }
1402            self.state.votes.push(record);
1403            self.state.save()?;
1404        }
1405        Ok(())
1406    }
1407
1408    // --------------------------------------------------------------- tally
1409
1410    fn tally(&mut self) -> Result<()> {
1411        if self.state.tally.is_some() {
1412            return Ok(());
1413        }
1414        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1415        let tops: Vec<char> = self
1416            .state
1417            .judgements
1418            .iter()
1419            .filter_map(|j| j.ranking.first().copied())
1420            .collect();
1421        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1422
1423        // A judge whose private vote failed still counted once, in the initial
1424        // ranking; using it beats discarding a whole seat.
1425        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1426        let mut cast: Vec<char> = Vec::new();
1427        for (i, j) in self.state.judgements.iter().enumerate() {
1428            let vote = self
1429                .state
1430                .votes
1431                .iter()
1432                .find(|v| v.judge == i + 1)
1433                .and_then(|v| v.vote)
1434                .or_else(|| j.ranking.first().copied());
1435            if let Some(v) = vote {
1436                *first_choice.entry(v).or_insert(0) += 1;
1437                cast.push(v);
1438            }
1439        }
1440
1441        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1442        for j in &self.state.judgements {
1443            let n = j.ranking.len();
1444            for (pos, label) in j.ranking.iter().enumerate() {
1445                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1446            }
1447        }
1448
1449        let best = first_choice.values().copied().max().unwrap_or(0);
1450        let mut leaders: Vec<char> = first_choice
1451            .iter()
1452            .filter(|(_, v)| **v == best)
1453            .map(|(k, _)| *k)
1454            .collect();
1455        let mut tie_break = None;
1456        if leaders.len() > 1 {
1457            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1458            let borda_leaders: Vec<char> = leaders
1459                .iter()
1460                .copied()
1461                .filter(|l| borda[l] == top_borda)
1462                .collect();
1463            tie_break = Some(if borda_leaders.len() == 1 {
1464                format!(
1465                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1466                    leaders.len()
1467                )
1468            } else {
1469                format!(
1470                    "{} way tie on both first-choice votes and Borda points, broken by label order",
1471                    leaders.len()
1472                )
1473            });
1474            leaders = borda_leaders;
1475            leaders.sort_unstable();
1476        }
1477        let winner = *leaders
1478            .first()
1479            .or(viable.first())
1480            .context("no candidate to declare a winner from")?;
1481
1482        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1483        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1484        let deliberated = !self.state.deliberation.is_empty();
1485
1486        // Whose verdict is this? A rate-limited seat is absent even if it
1487        // ranked before the limit hit, so presence is measured against the
1488        // recorded losses, not just "did a ranking ever appear".
1489        let quota_seats: std::collections::BTreeSet<&str> =
1490            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1491        let mut present = 0usize;
1492        for (i, j) in self.state.judgements.iter().enumerate() {
1493            if quota_seats.contains(j.seat.as_str()) {
1494                continue;
1495            }
1496            let ranked = !j.ranking.is_empty() && j.failed.is_none();
1497            let voted = self
1498                .state
1499                .votes
1500                .iter()
1501                .any(|v| v.judge == i + 1 && v.vote.is_some());
1502            if ranked || voted {
1503                present += 1;
1504            }
1505        }
1506        // Strict majority of the configured panel. A bare majority is real
1507        // signal we can act on, while a minority verdict must never stand in
1508        // for a healthy one. A one-candidate run needs no panel at all, and
1509        // `judges` stays `0` rather than the roster size a panel that never
1510        // sat would otherwise be credited with.
1511        let needs_quorum = viable.len() > 1;
1512        let judges_total = if needs_quorum {
1513            self.roles.judges.len()
1514        } else {
1515            0
1516        };
1517        let quorum = if needs_quorum {
1518            judges_total / 2 + 1
1519        } else {
1520            0
1521        };
1522        let met_quorum = !needs_quorum || present >= quorum;
1523        let uncontested = (!needs_quorum).then(|| {
1524            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1525        });
1526
1527        self.state.event(
1528            "tally",
1529            match &uncontested {
1530                Some(reason) => format!("winner {winner} — {reason}"),
1531                None => format!(
1532                    "winner {winner} — votes {} | initial {} | {} changed | \
1533                     {present}/{judges_total} judges{}",
1534                    first_choice
1535                        .iter()
1536                        .map(|(k, v)| format!("{k}:{v}"))
1537                        .collect::<Vec<_>>()
1538                        .join(" "),
1539                    if unanimous_initial {
1540                        "unanimous"
1541                    } else {
1542                        "split"
1543                    },
1544                    changed_votes,
1545                    if met_quorum {
1546                        String::new()
1547                    } else {
1548                        format!(" — below quorum ({quorum} required)")
1549                    },
1550                ),
1551            },
1552        );
1553        if !met_quorum {
1554            self.state.event(
1555                "stall",
1556                format!(
1557                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1558                     the run stops here, resumable"
1559                ),
1560            );
1561        }
1562        self.state.tally = Some(Tally {
1563            first_choice,
1564            borda,
1565            winner,
1566            rankings: tops.len(),
1567            unanimous_initial,
1568            deliberated,
1569            changed_votes,
1570            unanimous_final,
1571            tie_break,
1572            judges: judges_total,
1573            present,
1574            quorum,
1575            met_quorum,
1576            uncontested,
1577        });
1578        self.state.status = if met_quorum {
1579            RunStatus::Reviewing
1580        } else {
1581            RunStatus::Stalled
1582        };
1583        self.state.save()?;
1584        Ok(())
1585    }
1586
1587    // ------------------------------------------------------------- recover
1588
1589    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
1590    /// resumed toward completion once the transient cause clears.
1591    ///
1592    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
1593    /// it toward the quorum, which is exactly the set of seats whose absence
1594    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
1595    /// not depend on which node happened to hit the limit), or an ordinary
1596    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
1597    /// seat is never disturbed.
1598    ///
1599    /// A seat that now answers with a usable ranking is "recovered": its
1600    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
1601    /// `tally` counts it present again), and its vote re-collected. A seat that
1602    /// still fails keeps its loss and stays absent.
1603    ///
1604    /// Returns `true` when the re-tally restores the quorum (the run may proceed
1605    /// to review/gate/merge), `false` when it is still below quorum (the run
1606    /// stays `Stalled`, still resumable for a later retry).
1607    #[allow(clippy::too_many_lines)]
1608    async fn recover_stall(&mut self) -> Result<bool> {
1609        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1610        // agent files with `magi task add` name the run that paid for it. The
1611        // prompt overlay is cloned alongside it because the waves borrow it
1612        // while `self` is mutably borrowed by the node's own bookkeeping.
1613        let run_id = self.state.id.clone();
1614        let prompts = self.state.config.prompts.clone();
1615        // Absent seats = quota-lost at any node, or failed outright. Mirroring
1616        // `tally`'s presence test (rather than the old quota-judge/vote filter)
1617        // is what keeps a non-quota collapse — or a quota loss recorded at the
1618        // deliberate node — from being a permanent dead-end on `--resume`.
1619        let quota_seats: BTreeSet<&str> =
1620            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1621        let absent: Vec<String> = self
1622            .state
1623            .judgements
1624            .iter()
1625            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
1626            .map(|j| j.seat.clone())
1627            .collect();
1628        if absent.is_empty() {
1629            return Ok(false);
1630        }
1631        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1632        if viable.len() <= 1 {
1633            return Ok(false);
1634        }
1635        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1636        let language = self.state.config.graph.language.clone();
1637        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1638        let sessions = self.state.config.graph.sessions;
1639        let artifacts = agent::artifacts_dir(&self.state.dir());
1640        let root = self.state.worktree_root();
1641        let base_short = short(&self.state.base_commit);
1642        let candidates: Vec<Candidate> = viable.clone();
1643
1644        // Map each absent seat key to its 0-based position in `roles.judges`.
1645        let mut positions: Vec<usize> = absent
1646            .iter()
1647            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
1648            .collect();
1649        if positions.is_empty() {
1650            return Ok(false);
1651        }
1652        positions.sort_unstable();
1653        positions.dedup();
1654
1655        // Re-rank the lost seats, one blind prompt each.
1656        let mut judge_jobs = Vec::new();
1657        for &j in &positions {
1658            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1659            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1660            let seat_key = format!("judge-{}", j + 1);
1661            let spec = self.roles.judges[j].clone();
1662            let seat = self.seat(&seat_key, &spec.id);
1663            judge_jobs.push(SeatJob {
1664                spec,
1665                seat,
1666                prompt: prompt::judge(
1667                    &self.state.instruction,
1668                    &views,
1669                    self.roles.judges.len(),
1670                    &base_short,
1671                    &language,
1672                ),
1673                cwd: root.join(seat_key),
1674                timeout,
1675                allow_write: false,
1676                sessions,
1677                artifacts: artifacts.clone(),
1678                stem: format!("judge-{}-recover", j + 1),
1679            });
1680        }
1681
1682        let labels_for_check = labels.clone();
1683        let mut judge_losses = Vec::new();
1684        let retries = self.state.config.graph.retries;
1685        let cache = self.state.config.cache_dir();
1686        let ctx = WaveCtx {
1687            run: &run_id,
1688            node: "judge",
1689            prompts: &prompts,
1690            cache: cache.as_deref(),
1691        };
1692        let results = ask_json_wave::<Ranking>(
1693            judge_jobs,
1694            Arc::clone(&self.sem),
1695            retries,
1696            &ctx,
1697            &mut judge_losses,
1698            &mut self.state,
1699            &move |r: &Ranking| r.validate(&labels_for_check),
1700        )
1701        .await;
1702
1703        // Refresh the judgement of every seat that ranked again.
1704        let mut recovered: BTreeSet<usize> = BTreeSet::new();
1705        for (&j, (seat, res)) in positions.iter().zip(results) {
1706            self.state.seats.insert(seat.key.clone(), seat);
1707            let record = &mut self.state.judgements[j];
1708            match res {
1709                Ok((ranking, out)) => {
1710                    record.ranking = ranking.normalized();
1711                    record.reasons = ranking.reasons;
1712                    record.confidence = ranking.confidence;
1713                    record.failed = None;
1714                    record.duration_ms = out.duration_ms;
1715                    recovered.insert(j);
1716                    self.state.event(
1717                        "recover",
1718                        format!("judge {} ranked again after the limit", j + 1),
1719                    );
1720                }
1721                Err(e) => {
1722                    self.state
1723                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
1724                }
1725            }
1726        }
1727
1728        // Re-ask the votes of the seats that recovered a ranking.
1729        let mut vote_jobs = Vec::new();
1730        let mut vote_pos: Vec<usize> = Vec::new();
1731        for &j in &recovered {
1732            let seat_key = format!("judge-{}", j + 1);
1733            let spec = self.roles.judges[j].clone();
1734            let seat = self.seat(&seat_key, &spec.id);
1735            let mut text = prompt::final_vote(&labels, &language);
1736            if !has_context(&spec, &seat, sessions) {
1737                text = format!(
1738                    "{}\n\n# Candidates\n\n{}",
1739                    text,
1740                    self.candidate_block(&candidates, &base_short)
1741                );
1742            }
1743            vote_jobs.push(SeatJob {
1744                spec,
1745                seat,
1746                prompt: text,
1747                cwd: root.join(seat_key),
1748                timeout,
1749                allow_write: false,
1750                sessions,
1751                artifacts: artifacts.clone(),
1752                stem: format!("vote-judge-{}-recover", j + 1),
1753            });
1754            vote_pos.push(j);
1755        }
1756        let allowed = labels.clone();
1757        let mut vote_losses = Vec::new();
1758        let vote_retries = self.state.config.graph.retries;
1759        let vote_cache = self.state.config.cache_dir();
1760        let ctx = WaveCtx {
1761            run: &run_id,
1762            node: "vote",
1763            prompts: &prompts,
1764            cache: vote_cache.as_deref(),
1765        };
1766        let votes = ask_json_wave::<FinalVote>(
1767            vote_jobs,
1768            Arc::clone(&self.sem),
1769            vote_retries,
1770            &ctx,
1771            &mut vote_losses,
1772            &mut self.state,
1773            &move |v: &FinalVote| match v.label() {
1774                Some(c) if allowed.contains(&c) => Ok(()),
1775                other => bail!("vote {other:?} is not one of {allowed:?}"),
1776            },
1777        )
1778        .await;
1779        for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
1780            let agent_id = seat.agent.clone();
1781            self.state.seats.insert(seat.key.clone(), seat);
1782            match res {
1783                Ok((v, _)) => {
1784                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
1785                        rec.vote = v.label();
1786                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1787                    } else {
1788                        self.state.votes.push(VoteRecord {
1789                            judge: j + 1,
1790                            agent: agent_id,
1791                            vote: v.label(),
1792                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
1793                            changed: false,
1794                        });
1795                    }
1796                    self.state.event(
1797                        "recover",
1798                        format!("judge {} voted again after the limit", j + 1),
1799                    );
1800                }
1801                Err(e) => {
1802                    self.state
1803                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
1804                }
1805            }
1806        }
1807
1808        // A seat that ranked again is present even if its re-vote failed —
1809        // `tally` falls back to the initial ranking's first choice — so clear
1810        // its quota loss. Seats that still fail keep theirs and stay absent.
1811        if !recovered.is_empty() {
1812            let recovered_keys: BTreeSet<String> = recovered
1813                .iter()
1814                .map(|&j| format!("judge-{}", j + 1))
1815                .collect();
1816            self.state
1817                .quota
1818                .retain(|q| !recovered_keys.contains(&q.seat));
1819        }
1820
1821        // Recompute the verdict from the refreshed panel.
1822        self.state.tally = None;
1823        self.tally()?;
1824        Ok(self
1825            .state
1826            .tally
1827            .as_ref()
1828            .map(|t| t.met_quorum)
1829            .unwrap_or(false))
1830    }
1831
1832    // ----------------------------------------------------------------- fold
1833
1834    async fn fold_losers(&mut self) -> Result<()> {
1835        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
1836            return Ok(());
1837        };
1838        let repo = self.state.repo.clone();
1839        let mut folded = Vec::new();
1840        for i in 0..self.state.candidates.len() {
1841            let c = &self.state.candidates[i];
1842            if c.label == winner || c.folded {
1843                continue;
1844            }
1845            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
1846            git::worktree_remove(&repo, &wt).await.ok();
1847            git::branch_delete(&repo, &branch).await.ok();
1848            self.state.candidates[i].folded = true;
1849            folded.push(label.to_string());
1850        }
1851        // The judges are finished; their checkouts are pure cost from here.
1852        let root = self.state.worktree_root();
1853        for j in 1..=self.roles.judges.len() {
1854            let wt = root.join(format!("judge-{j}"));
1855            if wt.exists() {
1856                git::worktree_remove(&repo, &wt).await.ok();
1857            }
1858        }
1859        if !folded.is_empty() {
1860            self.state
1861                .event("fold", format!("folded candidates {}", folded.join(", ")));
1862            self.state.save()?;
1863        }
1864        Ok(())
1865    }
1866
1867    // ------------------------------------------------------------ base sync
1868
1869    /// Land the winner's tree on the current tip of `<remote>/<base>` before
1870    /// anything verifies it.
1871    ///
1872    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
1873    /// read whatever is checked out in the winner's worktree. Left alone that
1874    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
1875    /// it when the run *branched* - and a run takes long enough that the base
1876    /// has usually moved by the time it gets here. A gate that ran there
1877    /// answers "green on the commit this run started from", not "green on
1878    /// what is about to land", and the difference showed up three times in
1879    /// one day as a green run whose merge would have reverted a file another
1880    /// pull request had already landed.
1881    ///
1882    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
1883    /// implementation of the same idea: `land::Step::Rebase` already worked
1884    /// out the rules - throwaway worktree, conflict stops and reports rather
1885    /// than feeding a fixer, nothing runs in the primary tree - and a second
1886    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
1887    /// warns about ("two answers to a question nobody notices until a diff is
1888    /// wrong").
1889    ///
1890    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
1891    /// so it survives a park/resume. A conflict or a push failure sets
1892    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
1893    /// as they were - untouched, for a person to look at - which is also what
1894    /// makes re-entering this function afterwards a no-op instead of a second
1895    /// attempt at the same wall.
1896    async fn sync_to_base(&mut self) -> Result<()> {
1897        if self
1898            .state
1899            .base_sync
1900            .as_ref()
1901            .is_some_and(|s| s.conflict.is_some())
1902        {
1903            return Ok(());
1904        }
1905        let Some(winner) = self.state.winner().cloned() else {
1906            return Ok(());
1907        };
1908
1909        let repo = self.state.repo.clone();
1910        let remote = self.state.config.merge.remote.clone();
1911        let base_branch = self.state.base_branch.clone();
1912        let tracking = format!("{remote}/{base_branch}");
1913
1914        git::fetch(&repo, &remote, &base_branch).await.ok();
1915        // No network, or the remote never had this branch: `resolve_base`
1916        // already treats that as non-fatal at branch time, and a run that got
1917        // this far must not be blocked by it here either.
1918        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
1919            return Ok(());
1920        };
1921
1922        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1923        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
1924        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
1925
1926        if behind == 0 {
1927            self.state.base_sync = Some(BaseSync {
1928                tip,
1929                behind: 0,
1930                attempts,
1931                conflict: None,
1932            });
1933            self.state.save()?;
1934            return Ok(());
1935        }
1936
1937        if attempts >= BASE_SYNC_ROUNDS {
1938            let why = format!(
1939                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
1940                 rebase(s); rebasing again would only race it",
1941                winner.branch
1942            );
1943            self.state.status = RunStatus::Blocked;
1944            self.state.base_sync = Some(BaseSync {
1945                tip,
1946                behind,
1947                attempts,
1948                conflict: Some(why.clone()),
1949            });
1950            self.state.event("land", why);
1951            self.state.save()?;
1952            return Ok(());
1953        }
1954
1955        self.state.event(
1956            "land",
1957            format!(
1958                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
1959                winner.branch
1960            ),
1961        );
1962        self.state.save()?;
1963
1964        let scratch = self.state.dir().join("base-sync");
1965        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
1966        let attempts = attempts + 1;
1967        match rebased {
1968            Ok(None) => {
1969                // The branch ref moved, but a worktree that already had it
1970                // checked out (the winner's) was not told; sync its index and
1971                // files before anything reads them.
1972                git::sync_to_head(&winner.worktree).await?;
1973                self.state.base_sync = Some(BaseSync {
1974                    tip: tip.clone(),
1975                    behind: 0,
1976                    attempts,
1977                    conflict: None,
1978                });
1979                self.state
1980                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
1981            }
1982            Ok(Some(conflict)) => {
1983                let why = format!(
1984                    "{} conflicts with {tracking} and did not rebase: {}",
1985                    winner.branch,
1986                    conflict.chars().take(600).collect::<String>()
1987                );
1988                self.state.status = RunStatus::Blocked;
1989                self.state.base_sync = Some(BaseSync {
1990                    tip,
1991                    behind,
1992                    attempts,
1993                    conflict: Some(why.clone()),
1994                });
1995                self.state.event("land", why);
1996            }
1997            Err(e) => {
1998                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
1999                self.state.status = RunStatus::Blocked;
2000                self.state.base_sync = Some(BaseSync {
2001                    tip,
2002                    behind,
2003                    attempts,
2004                    conflict: Some(why.clone()),
2005                });
2006                self.state.event("land", why);
2007            }
2008        }
2009        self.state.save()?;
2010        Ok(())
2011    }
2012
2013    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
2014    /// last landed the winner on, once it has run, else the commit the run
2015    /// branched from.
2016    ///
2017    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
2018    /// and `vote` all happen before there is a winner to rebase, so they
2019    /// compare every candidate against the branch point on purpose, and a
2020    /// base that moves after they are already done cannot change an answer
2021    /// they already gave.
2022    fn landing_base(&self) -> String {
2023        self.state
2024            .base_sync
2025            .as_ref()
2026            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2027    }
2028
2029    // --------------------------------------------------------------- review
2030
2031    async fn review_loop(&mut self) -> Result<()> {
2032        // A base that would not rebase is a person's decision, not a review
2033        // round: nothing here would change the answer, and reviewers and a
2034        // fixer would be spending real budget on a tree that cannot land
2035        // regardless of what they find.
2036        if self
2037            .state
2038            .base_sync
2039            .as_ref()
2040            .is_some_and(|s| s.conflict.is_some())
2041        {
2042            return Ok(());
2043        }
2044        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2045        // agent files with `magi task add` name the run that paid for it. The
2046        // prompt overlay is cloned alongside it because the waves borrow it
2047        // while `self` is mutably borrowed by the node's own bookkeeping.
2048        let run_id = self.state.id.clone();
2049        let prompts = self.state.config.prompts.clone();
2050        let Some(winner) = self.state.winner().cloned() else {
2051            return Ok(());
2052        };
2053        let max_rounds = self.state.config.graph.review_rounds;
2054        // A clean round, an exhausted round budget, or a stalled tree (see
2055        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
2056        // they are recorded — recomputed here, not read off `status`, so a
2057        // reentry into a run that already stopped restates the identical
2058        // verdict instead of silently handing back whatever an earlier node
2059        // in this same walk clobbered `status` to (a solo-candidate
2060        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
2061        // below runs an empty range once the budget is spent, and would
2062        // otherwise fall through without touching `status` at all.
2063        if let Some(status) = review_conclusion(&self.state.reviews, max_rounds) {
2064            self.state.status = status;
2065            self.state.save()?;
2066            return Ok(());
2067        }
2068        self.state.status = RunStatus::Reviewing;
2069
2070        let repo = self.state.repo.clone();
2071        let root = self.state.worktree_root();
2072        let language = self.state.config.graph.language.clone();
2073        let sessions = self.state.config.graph.sessions;
2074        let artifacts = agent::artifacts_dir(&self.state.dir());
2075        let base = self.landing_base();
2076        let base_short = short(&base);
2077        let reviewers = self.roles.reviewers.clone();
2078        let shell = self.state.config.shell();
2079
2080        let mut prev_e2e: Option<String> = None;
2081        for round in (self.state.reviews.len() + 1)..=max_rounds {
2082            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2083            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2084            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2085
2086            // Each reviewer gets its own detached checkout of exactly this
2087            // commit: nobody can perturb the winner's tree, and the fixer can
2088            // keep working without racing a reviewer.
2089            let mut jobs = Vec::new();
2090            for (r, spec) in reviewers.iter().cloned().enumerate() {
2091                let wt = root.join(format!("review-{}", r + 1));
2092                if wt.exists() {
2093                    git::reset_detached(&wt, &head).await?;
2094                } else {
2095                    git::worktree_add_detached(&repo, &wt, &head).await?;
2096                }
2097                let seat_key = format!("review-{}", r + 1);
2098                let seat = self.seat(&seat_key, &spec.id);
2099                jobs.push(SeatJob {
2100                    prompt: prompt::review(&prompt::ReviewCtx {
2101                        instruction: &self.state.instruction,
2102                        branch: &winner.branch,
2103                        base_short: &base_short,
2104                        stat: &stat,
2105                        patch: &patch,
2106                        e2e: prev_e2e.as_deref(),
2107                        reviewers: reviewers.len(),
2108                        round,
2109                        rounds: max_rounds,
2110                        // A review-only run has no rankings, so nothing
2111                        // competed for this patch and the reviewer is told so.
2112                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2113                        language: &language,
2114                    }),
2115                    spec,
2116                    seat,
2117                    cwd: wt,
2118                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2119                    allow_write: false,
2120                    sessions,
2121                    artifacts: artifacts.clone(),
2122                    stem: format!("review-{round}-{}", r + 1),
2123                });
2124            }
2125
2126            self.state.event(
2127                "review",
2128                format!(
2129                    "round {round}: {} reviewers on {}",
2130                    jobs.len(),
2131                    short(&head)
2132                ),
2133            );
2134            let mut quota_losses = Vec::new();
2135            let review_retries = self.state.config.graph.retries;
2136            let review_cache = self.state.config.cache_dir();
2137            let ctx = WaveCtx {
2138                run: &run_id,
2139                node: "review",
2140                prompts: &prompts,
2141                cache: review_cache.as_deref(),
2142            };
2143            let results = ask_json_wave::<Review>(
2144                jobs,
2145                Arc::clone(&self.sem),
2146                review_retries,
2147                &ctx,
2148                &mut quota_losses,
2149                &mut self.state,
2150                &|_: &Review| Ok(()),
2151            )
2152            .await;
2153            self.state.quota.extend(quota_losses);
2154
2155            let mut records = Vec::new();
2156            let mut all_findings = Vec::new();
2157            for (r, (seat, res)) in results.into_iter().enumerate() {
2158                let agent_id = seat.agent.clone();
2159                self.state.seats.insert(seat.key.clone(), seat);
2160                let mut record = ReviewRecord {
2161                    reviewer: r + 1,
2162                    agent: agent_id,
2163                    summary: String::new(),
2164                    findings: Vec::new(),
2165                    failed: None,
2166                    duration_ms: 0,
2167                };
2168                match res {
2169                    Ok((review, out)) => {
2170                        record.summary = review.summary;
2171                        record.duration_ms = out.duration_ms;
2172                        for (n, mut f) in review.findings.into_iter().enumerate() {
2173                            // ids are magi's, never the agent's: the fixer's
2174                            // adoption report is keyed by them.
2175                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2176                            all_findings.push(f.clone());
2177                            record.findings.push(f);
2178                        }
2179                        self.state.event(
2180                            "review",
2181                            format!(
2182                                "round {round}: reviewer {} raised {} finding(s)",
2183                                r + 1,
2184                                record.findings.len()
2185                            ),
2186                        );
2187                    }
2188                    Err(e) => {
2189                        record.failed = Some(e.to_string());
2190                        self.state.event(
2191                            "review",
2192                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2193                        );
2194                    }
2195                }
2196                records.push(record);
2197            }
2198
2199            let mut e2e = run_commands(
2200                &shell,
2201                &self.state.config.verify.e2e,
2202                &winner.worktree,
2203                Duration::from_secs(self.state.config.graph.timeout_review),
2204            )
2205            .await;
2206            for o in &e2e {
2207                self.state.event(
2208                    "verify",
2209                    format!("round {round}: `{}` -> {}", o.command, e2e_outcome_label(o)),
2210                );
2211            }
2212
2213            // A build/link failure is not a verdict on the patch — it is
2214            // frequently a race against a shared `CARGO_TARGET_DIR` (see
2215            // AGENTS.md). Give verify one retry before letting a red like
2216            // that decide the round.
2217            let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
2218            if verify_retried {
2219                self.state.event(
2220                    "verify",
2221                    format!(
2222                        "round {round}: verify could not build/link, not a test result — \
2223                         retrying once before concluding"
2224                    ),
2225                );
2226                e2e = run_commands(
2227                    &shell,
2228                    &self.state.config.verify.e2e,
2229                    &winner.worktree,
2230                    Duration::from_secs(self.state.config.graph.timeout_review),
2231                )
2232                .await;
2233                for o in &e2e {
2234                    self.state.event(
2235                        "verify",
2236                        format!(
2237                            "round {round}: retry `{}` -> {}",
2238                            o.command,
2239                            e2e_outcome_label(o)
2240                        ),
2241                    );
2242                }
2243            }
2244
2245            let e2e_failures: String = e2e
2246                .iter()
2247                .filter(|o| !o.ok())
2248                .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2249                .collect();
2250
2251            let expected = records.len();
2252            let answered = records.iter().filter(|r| r.failed.is_none()).count();
2253            let incomplete = answered < expected;
2254            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2255            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
2256            let policy = self.state.config.graph.incomplete_review;
2257            let clean = round_is_clean(blocking, e2e_ok, answered, expected, policy);
2258
2259            let mut round_record = ReviewRound {
2260                round,
2261                head: head.clone(),
2262                reviews: records,
2263                e2e,
2264                verify_retried,
2265                fix: None,
2266                blocking,
2267                answered,
2268                expected,
2269                clean,
2270                progressed: false,
2271            };
2272
2273            if incomplete {
2274                let missing: Vec<String> = round_record
2275                    .reviews
2276                    .iter()
2277                    .filter(|r| r.failed.is_some())
2278                    .map(|r| format!("review-{}", r.reviewer))
2279                    .collect();
2280                self.state.event(
2281                    "review",
2282                    format!(
2283                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
2284                        missing.join(", ")
2285                    ),
2286                );
2287            }
2288
2289            if clean {
2290                self.state.event(
2291                    "review",
2292                    if incomplete {
2293                        format!(
2294                            "round {round}: clean (warn policy, incomplete panel) — no \
2295                             blocking findings from the seats that answered, verification green"
2296                        )
2297                    } else {
2298                        format!("round {round}: clean — no blocking findings, verification green")
2299                    },
2300                );
2301                self.state.reviews.push(round_record);
2302                self.state.status = RunStatus::Gating;
2303                self.state.save()?;
2304                return Ok(());
2305            }
2306
2307            // Nothing was raised and verification passed, but not every seat
2308            // answered and the policy refuses to call that clean: re-review
2309            // rather than send the fixer after a round with nothing to fix.
2310            if incomplete && blocking == 0 && e2e_ok {
2311                self.state.reviews.push(round_record);
2312                self.state.save()?;
2313                if round == max_rounds {
2314                    self.state.status = RunStatus::Blocked;
2315                    self.state.event(
2316                        "review",
2317                        format!(
2318                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
2319                             refusing to call it clean",
2320                            expected - answered
2321                        ),
2322                    );
2323                    return Ok(());
2324                }
2325                prev_e2e = None;
2326                continue;
2327            }
2328
2329            if round == max_rounds {
2330                self.state.reviews.push(round_record);
2331                return self.stop_reviewing(&format!(
2332                    "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
2333                ));
2334            }
2335
2336            // Fix. The winner's own implementer seat continues its conversation:
2337            // the competition is over, so context is pure benefit now.
2338            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
2339                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
2340                _ => (
2341                    self.state
2342                        .config
2343                        .agent(&winner.agent)
2344                        .cloned()
2345                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
2346                    format!("impl-{}", winner.label),
2347                ),
2348            };
2349            let seat = self.seat(&fix_seat_key, &fix_spec.id);
2350            let blocking_findings: Vec<_> = all_findings
2351                .iter()
2352                .filter(|f| f.severity.blocks())
2353                .cloned()
2354                .collect();
2355            let job = SeatJob {
2356                prompt: prompt::fix(
2357                    &self.state.instruction,
2358                    &blocking_findings,
2359                    (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
2360                    round,
2361                    max_rounds,
2362                    &language,
2363                ),
2364                spec: fix_spec.clone(),
2365                seat,
2366                cwd: winner.worktree.clone(),
2367                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
2368                allow_write: true,
2369                sessions,
2370                artifacts: artifacts.clone(),
2371                stem: format!("fix-{round}"),
2372            };
2373            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
2374            let cache = self.state.config.cache_dir();
2375            let ctx = WaveCtx {
2376                run: &run_id,
2377                node: "fix",
2378                prompts: &prompts,
2379                cache: cache.as_deref(),
2380            };
2381            let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2382            let agent_id = seat.agent.clone();
2383            let seat_key = seat.key.clone();
2384            self.state.seats.insert(seat.key.clone(), seat);
2385
2386            let mut fix = FixRecord {
2387                agent: agent_id,
2388                addressed: Vec::new(),
2389                rejected: Vec::new(),
2390                notes: String::new(),
2391                committed: false,
2392                failed: None,
2393                duration_ms: 0,
2394            };
2395            match out {
2396                AgentOutcome::Ok(o) => {
2397                    fix.duration_ms = o.duration_ms;
2398                    match verdict::extract_json::<FixReport>(&o.text) {
2399                        Ok(report) => {
2400                            fix.addressed = report.addressed;
2401                            fix.rejected = report.rejected;
2402                            fix.notes =
2403                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
2404                        }
2405                        Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
2406                    }
2407                }
2408                // The CLI's raw error JSON is not a fix report to parse.
2409                AgentOutcome::Dropped(o) => {
2410                    fix.duration_ms = o.duration_ms;
2411                    let why = o
2412                        .dropped
2413                        .as_ref()
2414                        .map(|d| d.why.as_str())
2415                        .unwrap_or("the CLI ended the stream without delivering its answer");
2416                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
2417                }
2418                AgentOutcome::Quota(o) => {
2419                    self.state.quota.push(QuotaLoss {
2420                        seat: seat_key,
2421                        node: "fix".to_owned(),
2422                        at: Timestamp::now(),
2423                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2424                    });
2425                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
2426                }
2427                AgentOutcome::Failed(e) => fix.failed = Some(e),
2428            }
2429            git::commit_all(
2430                &winner.worktree,
2431                &format!("magi: review round {round} fixes (uncommitted work)"),
2432            )
2433            .await
2434            .ok();
2435            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
2436            fix.committed = after != before;
2437            // Judged by what `git` says moved against base, never by the
2438            // fixer's own `addressed`/`rejected` count — see
2439            // `ReviewRound::progressed`. Propagated with `?`, the same as the
2440            // `patch` snapshot above: swallowing this error would default
2441            // `diff_after` to empty, which almost always differs from a
2442            // non-empty `patch` and reads as "progressed" — exactly backwards
2443            // for a `git` failure the stagnation check cannot see through.
2444            let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
2445            let progressed = diff_after != patch;
2446            let commit_note = if fix.committed {
2447                "committed"
2448            } else {
2449                "NO new commit"
2450            };
2451            let tree_note = if progressed {
2452                "changed vs base"
2453            } else {
2454                "unchanged vs base"
2455            };
2456            self.state.event(
2457                "fix",
2458                match &fix.failed {
2459                    // Distinct on purpose from "0 addressed, 0 rejected": the
2460                    // fixer's own diff still landed (blocking counts do keep
2461                    // falling round over round), only its adoption report did
2462                    // not come back, so this must never read like every
2463                    // finding was reviewed and declined.
2464                    Some(reason) => {
2465                        format!(
2466                            "round {round}: fixer's adoption report was lost ({reason}); \
2467                             {commit_note}, tree {tree_note}"
2468                        )
2469                    }
2470                    None => format!(
2471                        "round {round}: {} addressed, {} rejected, {commit_note}, tree {tree_note}",
2472                        fix.addressed.len(),
2473                        fix.rejected.len(),
2474                    ),
2475                },
2476            );
2477            round_record.fix = Some(fix);
2478            round_record.progressed = progressed;
2479            self.state.reviews.push(round_record);
2480            self.state.save()?;
2481
2482            prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
2483
2484            let streak = self
2485                .state
2486                .reviews
2487                .iter()
2488                .rev()
2489                .take_while(|r| !r.progressed)
2490                .count();
2491            if streak >= STAGNANT_LIMIT {
2492                return self.stop_reviewing(&format!(
2493                    "the tree has not moved against base for {streak} round(s) in a row"
2494                ));
2495            }
2496        }
2497        Ok(())
2498    }
2499
2500    /// Decide, from the last recorded round's own verification, whether
2501    /// stopping the review loop is a hand-off or a genuine block.
2502    ///
2503    /// Called once the loop has given up trying — the round budget is spent,
2504    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
2505    /// findings still open, never while a round is still clean or the
2506    /// incomplete-panel case handled inline above. Gate and e2e are facts
2507    /// about the tree; a lingering review finding is an opinion, and this
2508    /// workload's own `magi stats` puts reviewer precision low enough
2509    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
2510    /// must not by itself stand between a green, verified change and the
2511    /// human who decides what to do with it. A red e2e is not an opinion, so
2512    /// that case still blocks, with the failing command and a tail of its
2513    /// output recorded here rather than left in `run.json` for someone to go
2514    /// find.
2515    fn stop_reviewing(&mut self, why: &str) -> Result<()> {
2516        let last = self
2517            .state
2518            .reviews
2519            .last()
2520            .expect("a round was just recorded before this is called");
2521        let red: Vec<String> = last
2522            .e2e
2523            .iter()
2524            .filter(|o| !o.ok())
2525            .map(|o| {
2526                format!(
2527                    "`{}` -> {:?}\n{}",
2528                    o.command,
2529                    o.code,
2530                    tail(&o.output_tail, EVENT_OUTPUT_TAIL)
2531                )
2532            })
2533            .collect();
2534        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
2535
2536        if red.is_empty() {
2537            self.state.event(
2538                "review",
2539                format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
2540            );
2541            self.state.status = RunStatus::Gating;
2542        } else {
2543            self.state
2544                .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
2545            self.state.status = RunStatus::Blocked;
2546        }
2547        self.state.save()?;
2548        Ok(())
2549    }
2550
2551    // ----------------------------------------------------------------- gate
2552
2553    async fn gate(&mut self) -> Result<()> {
2554        // Judged by the review record itself, not by `status`: a solo
2555        // candidate's `judge`/`deliberate` skip rewrites `status` on every
2556        // reentry (see `judge`), and trusting it here is exactly how a run
2557        // that exhausted its review budget got gated and merged a second
2558        // time around. `review_conclusion` recomputes the review loop's own
2559        // verdict from the round records themselves — `Gating` for a clean
2560        // round or a hand-off (see `stop_reviewing`), anything else means the
2561        // loop is still going or genuinely blocked.
2562        // A base the winner could not be replayed onto is a decision, not a
2563        // round: there is no landing tree to gate. Read as its own record for
2564        // the same reason the review verdict is.
2565        if self.state.status == RunStatus::Failed
2566            || self
2567                .state
2568                .base_sync
2569                .as_ref()
2570                .is_some_and(|s| s.conflict.is_some())
2571            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
2572                != Some(RunStatus::Gating)
2573        {
2574            return Ok(());
2575        }
2576        if !self.state.gate.is_empty() {
2577            return Ok(());
2578        }
2579        let Some(winner) = self.state.winner().cloned() else {
2580            return Ok(());
2581        };
2582        self.state.status = RunStatus::Gating;
2583        let shell = self.state.config.shell();
2584        let outcomes = run_commands(
2585            &shell,
2586            &self.state.config.verify.gate,
2587            &winner.worktree,
2588            Duration::from_secs(self.state.config.graph.timeout_review),
2589        )
2590        .await;
2591        for o in &outcomes {
2592            self.state.event(
2593                "gate",
2594                format!(
2595                    "`{}` -> {}",
2596                    o.command,
2597                    if o.ok() {
2598                        "pass".to_owned()
2599                    } else {
2600                        format!(
2601                            "FAIL ({:?})\n{}",
2602                            o.code,
2603                            tail(&o.output_tail, EVENT_OUTPUT_TAIL)
2604                        )
2605                    }
2606                ),
2607            );
2608        }
2609        let passed = outcomes.iter().all(CommandOutcome::ok);
2610        self.state.gate = outcomes;
2611        if !passed {
2612            self.state.status = RunStatus::Blocked;
2613            self.state.event("gate", "gate failed; not merging");
2614        }
2615        self.state.save()?;
2616        Ok(())
2617    }
2618
2619    // ---------------------------------------------------------------- merge
2620
2621    async fn merge(&mut self) -> Result<()> {
2622        // Same reasoning as `gate`: ask the review and gate records directly
2623        // rather than `status`, which a solo-candidate `judge`/`deliberate`
2624        // skip can rewrite on reentry to something that no longer says
2625        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
2626        // so a hand-off (open findings, green verification) reaches merge
2627        // exactly like a genuinely clean round does.
2628        if self
2629            .state
2630            .base_sync
2631            .as_ref()
2632            .is_some_and(|s| s.conflict.is_some())
2633            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
2634                != Some(RunStatus::Gating)
2635            || self.state.gate.iter().any(|o| !o.ok())
2636        {
2637            return Ok(());
2638        }
2639        // This node's own record, not `status`: `status == Ready` is not
2640        // unique to the harmless `MergeMode::None` path this line was
2641        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
2642        // run's PR was closed without merging — and on that run `mode` is
2643        // still `Pr`, so a reentry that fell through here would push and
2644        // open a second pull request. `self.state.merge` is set exactly once
2645        // this node (or `land`) has already produced a verdict, under every
2646        // mode, which is what "already done" actually means here.
2647        if self.state.merge.is_some() {
2648            return Ok(());
2649        }
2650        let Some(winner) = self.state.winner().cloned() else {
2651            return Ok(());
2652        };
2653        let repo = self.state.repo.clone();
2654        let base = self.state.base_branch.clone();
2655        let mode = self.state.config.merge.mode;
2656        let message = pr_body(&self.state, winner.label);
2657
2658        let outcome = match mode {
2659            MergeMode::None => MergeOutcome {
2660                mode,
2661                ok: true,
2662                detail: format!("git -C {} merge --no-ff {}", repo.display(), winner.branch),
2663            },
2664            MergeMode::Local => {
2665                let on = git::current_branch(&repo).await?;
2666                if on.as_deref() != Some(base.as_str()) {
2667                    MergeOutcome {
2668                        mode,
2669                        ok: false,
2670                        detail: format!(
2671                            "{} has {} checked out, not the base branch {base}",
2672                            repo.display(),
2673                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
2674                        ),
2675                    }
2676                } else if !git::is_clean(&repo).await? {
2677                    MergeOutcome {
2678                        mode,
2679                        ok: false,
2680                        detail: format!("{} is dirty; refusing to merge", repo.display()),
2681                    }
2682                } else {
2683                    let out = git::merge_no_ff(&repo, &winner.branch, &message).await?;
2684                    MergeOutcome {
2685                        mode,
2686                        ok: out.ok(),
2687                        detail: if out.ok() { out.stdout } else { out.stderr },
2688                    }
2689                }
2690            }
2691            MergeMode::Pr => {
2692                let remote = self.state.config.merge.remote.clone();
2693                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
2694                if !pushed.ok() {
2695                    MergeOutcome {
2696                        mode,
2697                        ok: false,
2698                        detail: pushed.stderr,
2699                    }
2700                } else {
2701                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
2702                    match out {
2703                        Ok(url) => MergeOutcome {
2704                            mode,
2705                            ok: true,
2706                            detail: url,
2707                        },
2708                        Err(e) => MergeOutcome {
2709                            mode,
2710                            ok: false,
2711                            detail: e.to_string(),
2712                        },
2713                    }
2714                }
2715            }
2716        };
2717
2718        self.state.status = match (mode, outcome.ok) {
2719            (MergeMode::None, _) => RunStatus::Ready,
2720            (_, true) => RunStatus::Merged,
2721            (_, false) => RunStatus::Blocked,
2722        };
2723        self.state.event(
2724            "merge",
2725            format!(
2726                "{:?}: {}",
2727                mode,
2728                outcome.detail.lines().next().unwrap_or("")
2729            ),
2730        );
2731        self.state.merge = Some(outcome);
2732        self.state.save()?;
2733
2734        // The PR is open and the run would historically stop here, leaving the
2735        // operator to watch checks, feed review comments back to a fixer, and
2736        // merge. That was done by hand six times in one session before this
2737        // existed. Opt-in, because merging is the one irreversible thing magi
2738        // can do to a repository.
2739        if self.state.config.graph.land
2740            && mode == MergeMode::Pr
2741            && self.state.status == RunStatus::Merged
2742        {
2743            let url = self
2744                .state
2745                .merge
2746                .as_ref()
2747                .map(|m| m.detail.clone())
2748                .unwrap_or_default();
2749            let url = url.lines().next().unwrap_or("").trim().to_owned();
2750            if url.starts_with("http") {
2751                // A land failure is not a lost run: the work is on a branch and
2752                // the PR is open, which is exactly where a human takes over.
2753                match land::land(&mut self.state, &url).await {
2754                    Ok(pr) => {
2755                        self.state.status = match pr.state {
2756                            land::PrLifecycle::Merged => RunStatus::Merged,
2757                            _ => RunStatus::Blocked,
2758                        };
2759                    }
2760                    Err(e) => {
2761                        self.state.status = RunStatus::Blocked;
2762                        self.state.event("land", format!("gave up: {e}"));
2763                    }
2764                }
2765                self.state.save()?;
2766            }
2767        }
2768        Ok(())
2769    }
2770
2771    // -------------------------------------------------------------- helpers
2772
2773    /// Fetch or create a seat, keeping its conversation across nodes.
2774    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
2775        if let Some(existing) = self.state.seats.get(key)
2776            && existing.agent == agent
2777        {
2778            return existing.clone();
2779        }
2780        let fresh = SeatState::new(key, agent, self.state.seed);
2781        self.state.seats.insert(key.to_owned(), fresh.clone());
2782        fresh
2783    }
2784
2785    /// A candidate rendered for judging, with the leak policy applied.
2786    fn view(&self, c: &Candidate) -> CandidateView {
2787        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2788            .unwrap_or_default();
2789        let (patch, _) = blind::sanitize_patch(
2790            &format!("candidate {} patch", c.label),
2791            &raw,
2792            &self.state.config.blind,
2793        );
2794        CandidateView {
2795            label: c.label,
2796            branch: c.branch.clone(),
2797            summary: c.summary.clone(),
2798            stat: c.stat.clone(),
2799            patch,
2800        }
2801    }
2802
2803    /// The full candidate set as prompt text, for seats with no live session.
2804    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
2805        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
2806        prompt::judge(
2807            "(see above)",
2808            &views,
2809            self.roles.judges.len(),
2810            base_short,
2811            "en",
2812        )
2813    }
2814
2815    /// Anonymised transcript for judge `self_idx`.
2816    ///
2817    /// The initial rankings are always the opening statements. Seeding them
2818    /// only when no turn had been taken yet meant every judge after the first
2819    /// argued against a single voice instead of against the actual split — the
2820    /// disagreement is the information, so it is always on the table.
2821    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
2822        let mut turns = Vec::new();
2823        for j in &self.state.judgements {
2824            if j.ranking.is_empty() {
2825                continue;
2826            }
2827            let reasons = j
2828                .reasons
2829                .iter()
2830                .map(|(k, v)| format!("- {k}: {v}"))
2831                .collect::<Vec<_>>()
2832                .join("\n");
2833            turns.push(Turn {
2834                who: format!("Judge {} (opening ranking)", j.judge),
2835                is_self: j.judge == self_idx + 1,
2836                body: format!(
2837                    "Ranked {}{}{reasons}",
2838                    j.ranking.iter().collect::<String>(),
2839                    if reasons.is_empty() {
2840                        ""
2841                    } else {
2842                        ", because:\n"
2843                    }
2844                ),
2845            });
2846        }
2847        for t in self
2848            .state
2849            .deliberation
2850            .iter()
2851            .flat_map(|r| r.turns.iter())
2852            .chain(current)
2853        {
2854            turns.push(Turn {
2855                who: format!("Judge {}", t.judge),
2856                is_self: t.judge == self_idx + 1,
2857                body: t.body.clone(),
2858            });
2859        }
2860        turns
2861    }
2862}
2863
2864/// Does this seat still hold the context a follow-up prompt would rely on?
2865fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
2866    agent::has_session(spec.kind, seat, sessions)
2867}
2868
2869fn short(commit: &str) -> String {
2870    commit.chars().take(7).collect()
2871}
2872
2873fn make_executable(path: &Path) -> Result<()> {
2874    #[cfg(unix)]
2875    {
2876        use std::os::unix::fs::PermissionsExt as _;
2877        let mut perms = std::fs::metadata(path)?.permissions();
2878        perms.set_mode(0o755);
2879        std::fs::set_permissions(path, perms)?;
2880    }
2881    #[cfg(not(unix))]
2882    {
2883        let _ = path;
2884    }
2885    Ok(())
2886}
2887
2888/// What every seat in one batch shares: where the answers are attributed, the
2889/// prompt overlay they inherit, and the build cache they are told to use.
2890///
2891/// A struct rather than four more parameters: `wave` also needs the run's
2892/// state (to record who is answering right now) and the attempt number, and
2893/// eight positional arguments is both unreadable and a clippy error.
2894struct WaveCtx<'a> {
2895    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
2896    /// paid for it.
2897    run: &'a str,
2898    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
2899    node: &'a str,
2900    prompts: &'a Prompts,
2901    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
2902    cache: Option<&'a Path>,
2903}
2904
2905/// Run one job, honouring the parallelism budget.
2906async fn run_one(
2907    job: SeatJob,
2908    sem: Arc<Semaphore>,
2909    ctx: &WaveCtx<'_>,
2910    state: &mut RunState,
2911    attempt: usize,
2912) -> (SeatState, AgentOutcome) {
2913    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
2914        .await
2915        .pop()
2916        .expect("one job in, one result out");
2917    (seat, out)
2918}
2919
2920/// Run every job concurrently, capped by the semaphore, preserving order.
2921///
2922/// Every seat in the batch is recorded into [`RunState::active`] before the
2923/// wave starts and cleared as each answer lands, so the run's own record says
2924/// who is still being waited on rather than only who finished.
2925async fn wave(
2926    jobs: Vec<SeatJob>,
2927    sem: Arc<Semaphore>,
2928    ctx: &WaveCtx<'_>,
2929    state: &mut RunState,
2930    attempt: usize,
2931) -> Vec<(usize, SeatState, AgentOutcome)> {
2932    let WaveCtx {
2933        run,
2934        node,
2935        prompts,
2936        cache,
2937    } = *ctx;
2938    for job in &jobs {
2939        state.seat_started(node, &job.seat.key, job.timeout, attempt);
2940    }
2941    if let Err(e) = state.save() {
2942        // A failed persist of "who is answering right now" must not abort the
2943        // wave: the seats are already being asked, and the alternative is
2944        // losing the answers to save a status line nobody may even be
2945        // watching.
2946        tracing::warn!("could not persist in-progress seats: {e:#}");
2947    }
2948    let mut set = tokio::task::JoinSet::new();
2949    let overlay = prompts.overlay(node);
2950    for (i, mut job) in jobs.into_iter().enumerate() {
2951        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
2952        if cache.is_some() {
2953            job.prompt.push('\n');
2954            job.prompt.push_str(prompt::build_cache_note());
2955        }
2956        let sem = Arc::clone(&sem);
2957        let run = run.to_owned();
2958        let node = node.to_owned();
2959        let cache = cache.map(Path::to_path_buf);
2960        set.spawn(async move {
2961            let _permit = sem.acquire().await;
2962            let mut seat = job.seat;
2963            let out = agent::invoke(
2964                &job.spec,
2965                &mut seat,
2966                &Invocation {
2967                    cwd: &job.cwd,
2968                    prompt: &job.prompt,
2969                    timeout: job.timeout,
2970                    allow_write: job.allow_write,
2971                    sessions: job.sessions,
2972                    artifacts: &job.artifacts,
2973                    stem: &job.stem,
2974                    run: &run,
2975                    node: &node,
2976                    cache_dir: cache.as_deref(),
2977                },
2978            )
2979            .await;
2980            let out = match out {
2981                Ok(o) if o.usable() => AgentOutcome::Ok(o),
2982                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
2983                // Billed work the CLI failed to hand over is not an ordinary
2984                // failure, but its text is the CLI's raw error JSON, not an
2985                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
2986                // read it as one by forgetting to check. `usable()` is always
2987                // false here (dropped implies an empty response), so this has
2988                // to be checked before the catch-all `Failed` below or the
2989                // one shape this exists for is lost with the rest.
2990                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
2991                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
2992                Ok(o) => AgentOutcome::Failed(format!(
2993                    "exited with {:?} and no usable output",
2994                    o.exit_code
2995                )),
2996                Err(e) => AgentOutcome::Failed(e.to_string()),
2997            };
2998            (i, seat, out)
2999        });
3000    }
3001    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
3002    while let Some(joined) = set.join_next().await {
3003        let (i, seat, out) = match joined {
3004            Ok(v) => v,
3005            // No seat to clear: a panicked task never reported which one it
3006            // was. The defensive sweep below this loop is what stops that
3007            // seat's `active` entry from surviving forever.
3008            Err(e) => {
3009                tracing::error!("agent task panicked: {e}");
3010                continue;
3011            }
3012        };
3013        state.seat_finished(&seat.key);
3014        if let Err(e) = state.save() {
3015            tracing::warn!("could not persist a seat's completion: {e:#}");
3016        }
3017        if collected.len() <= i {
3018            collected.resize_with(i + 1, || None);
3019        }
3020        collected[i] = Some((i, seat, out));
3021    }
3022    // Belt-and-braces for the panic branch above: every seat this exact batch
3023    // started shares this `(node, attempt)` pair, and every seat that finished
3024    // normally already cleared itself, so anything left tagged with it here
3025    // can only be a panicked task's leftover. Cleared unconditionally rather
3026    // than left to read as still answering forever.
3027    if state
3028        .active
3029        .values()
3030        .any(|a| a.node == node && a.attempt == attempt)
3031    {
3032        state
3033            .active
3034            .retain(|_, a| !(a.node == node && a.attempt == attempt));
3035        if let Err(e) = state.save() {
3036            tracing::warn!("could not persist the end of a wave: {e:#}");
3037        }
3038    }
3039    collected.into_iter().flatten().collect()
3040}
3041
3042/// Is a review round clean, given how many reviewer seats answered against
3043/// how many the round expected?
3044///
3045/// A seat that never answered (timeout, crash, unparsable output) is not a
3046/// seat that read the patch and found nothing — treating it as such is
3047/// exactly the bug this function exists to close. Under the default `block`
3048/// policy a missing seat can never be clean; `warn` still requires the seats
3049/// that *did* answer to have found nothing blocking and verification to be
3050/// green.
3051fn round_is_clean(
3052    blocking: usize,
3053    e2e_ok: bool,
3054    answered: usize,
3055    expected: usize,
3056    policy: IncompleteReviewPolicy,
3057) -> bool {
3058    blocking == 0 && e2e_ok && (answered == expected || policy == IncompleteReviewPolicy::Warn)
3059}
3060
3061/// The review loop's own conclusion, derived entirely from its persisted
3062/// round records and the round budget that produced them — never from
3063/// `status`, so a reentry (or `gate`/`merge` reading it independently)
3064/// recomputes the identical answer regardless of what an earlier node in the
3065/// same walk, or a previous walk, did to `status`.
3066///
3067/// `None` while more rounds remain to try, including when review never ran
3068/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
3069/// gone clean, or the budget is spent, or the tree has stopped moving (see
3070/// [`STAGNANT_LIMIT`]), the answer is one of two things:
3071///
3072/// - An incomplete panel that raised nothing is missing input, not a
3073///   verified tree — never a hand-off candidate, whatever verification said
3074///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
3075/// - Otherwise, green e2e on the last round hands off (see
3076///   [`Runner::stop_reviewing`]); red e2e blocks.
3077fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
3078    if max_rounds == 0 || reviews.iter().any(|r| r.clean) {
3079        return Some(RunStatus::Gating);
3080    }
3081    let last = reviews.last()?;
3082    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
3083    if reviews.len() < max_rounds && !stagnant {
3084        return None;
3085    }
3086    Some(if last.incomplete() && last.blocking == 0 {
3087        RunStatus::Blocked
3088    } else if last.e2e.iter().all(CommandOutcome::ok) {
3089        RunStatus::Gating
3090    } else {
3091        RunStatus::Blocked
3092    })
3093}
3094
3095/// How long a re-ask may take, given the budget the first attempt had.
3096///
3097/// A `nudged` retry is a request to restate an answer the seat has already
3098/// worked out: it carries no new work, so it does not deserve the original
3099/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
3100/// 133 seconds while a third sat for over ten minutes on a resumed session
3101/// holding 410 KB of prior output - and because the retry had inherited the
3102/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
3103/// judging round whose other seats were long finished.
3104///
3105/// A quarter of the budget, with a floor so that a deliberately short timeout
3106/// does not collapse to nothing. A retry that re-sends the whole prompt
3107/// (because the seat kept no context) is the original job again, and keeps the
3108/// original budget.
3109fn retry_budget(full: Duration, nudged: bool) -> Duration {
3110    if nudged {
3111        (full / 4).max(Duration::from_secs(120)).min(full)
3112    } else {
3113        full
3114    }
3115}
3116
3117/// Run a wave and parse each reply, re-asking the seats whose reply was
3118/// unusable.
3119///
3120/// The re-ask is a nudge rather than the whole prompt again when the seat still
3121/// holds its conversation, which is the difference between a cheap retry and
3122/// paying for the entire candidate set twice.
3123///
3124/// A seat that hits a rate limit is **not** re-asked: the same call will fail
3125/// the same way until the limit resets, so spending a retry attempt on it is
3126/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
3127/// like any other absent seat — the caller decides whether the panel still has
3128/// a quorum.
3129#[allow(clippy::too_many_arguments)]
3130async fn ask_json_wave<T>(
3131    jobs: Vec<SeatJob>,
3132    sem: Arc<Semaphore>,
3133    retries: usize,
3134    ctx: &WaveCtx<'_>,
3135    losses: &mut Vec<QuotaLoss>,
3136    state: &mut RunState,
3137    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
3138) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
3139where
3140    T: serde::de::DeserializeOwned + Send + 'static,
3141{
3142    let n = jobs.len();
3143    let originals: Vec<SeatJob> = jobs;
3144    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
3145    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
3146    let mut pending: Vec<usize> = (0..n).collect();
3147
3148    for attempt in 0..=retries {
3149        if pending.is_empty() {
3150            break;
3151        }
3152        let mut batch = Vec::with_capacity(pending.len());
3153        for &i in &pending {
3154            let src = &originals[i];
3155            // The prompt and the budget are one decision: a nudge restates
3156            // finished work, a re-sent prompt redoes it.
3157            let (prompt, timeout) = if attempt == 0 {
3158                (src.prompt.clone(), src.timeout)
3159            } else {
3160                let why = done[i]
3161                    .as_ref()
3162                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
3163                    .unwrap_or_else(|| "no parsable answer".to_owned());
3164                let nudge = prompt::nudge(&why);
3165                let nudged = has_context(&src.spec, &seats[i], src.sessions);
3166                let prompt = if nudged {
3167                    nudge
3168                } else {
3169                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
3170                };
3171                (prompt, retry_budget(src.timeout, nudged))
3172            };
3173            batch.push(SeatJob {
3174                spec: src.spec.clone(),
3175                seat: seats[i].clone(),
3176                cwd: src.cwd.clone(),
3177                prompt,
3178                timeout,
3179                allow_write: src.allow_write,
3180                sessions: src.sessions,
3181                artifacts: src.artifacts.clone(),
3182                stem: if attempt == 0 {
3183                    src.stem.clone()
3184                } else {
3185                    format!("{}-retry{attempt}", src.stem)
3186                },
3187            });
3188        }
3189
3190        if attempt > 0 {
3191            let seats_out: Vec<&str> = pending
3192                .iter()
3193                .map(|&i| originals[i].seat.key.as_str())
3194                .collect();
3195            state.event(
3196                ctx.node,
3197                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3198            );
3199        }
3200        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3201        let mut still = Vec::new();
3202        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3203            seats[i] = seat;
3204            let (parsed, quota) = match out {
3205                AgentOutcome::Ok(o) => (
3206                    match verdict::extract_json::<T>(&o.text) {
3207                        Ok(v) => match validate(&v) {
3208                            Ok(()) => Ok((v, o)),
3209                            Err(e) => Err(e),
3210                        },
3211                        Err(e) => Err(e),
3212                    },
3213                    false,
3214                ),
3215                AgentOutcome::Quota(o) => {
3216                    losses.push(QuotaLoss {
3217                        seat: originals[i].seat.key.clone(),
3218                        node: ctx.node.to_owned(),
3219                        at: Timestamp::now(),
3220                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3221                    });
3222                    (
3223                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
3224                        true,
3225                    )
3226                }
3227                // Not a parseable answer, but also not worth a special-cased
3228                // retry here: the nudge loop above already re-asks anything
3229                // that fails to parse, which is exactly what a dropped stream
3230                // needs. Just don't hand its raw error JSON to `extract_json`.
3231                AgentOutcome::Dropped(o) => {
3232                    let why = o
3233                        .dropped
3234                        .as_ref()
3235                        .map(|d| d.why.as_str())
3236                        .unwrap_or("the CLI ended the stream without delivering its answer");
3237                    (
3238                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
3239                        false,
3240                    )
3241                }
3242                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
3243            };
3244            let failed = parsed.is_err();
3245            done[i] = Some(parsed);
3246            // Do not re-ask a rate-limited seat (quota) — a retry is known to
3247            // fail the same way; and never re-ask a seat that already parsed.
3248            if failed && !quota {
3249                still.push(i);
3250            }
3251        }
3252        pending = still;
3253    }
3254
3255    seats
3256        .into_iter()
3257        .zip(done)
3258        .map(|(seat, res)| {
3259            (
3260                seat,
3261                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
3262            )
3263        })
3264        .collect()
3265}
3266
3267/// Describe one verify command's outcome for the event log, distinguishing a
3268/// build/link failure — the toolchain never produced a binary to run — from
3269/// an actual test failure, since only the latter is a verdict on the patch.
3270fn e2e_outcome_label(o: &CommandOutcome) -> String {
3271    if o.ok() {
3272        return "pass".to_owned();
3273    }
3274    let reason = if o.build_failed() {
3275        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
3276    } else {
3277        format!("FAIL ({:?})", o.code)
3278    };
3279    format!("{reason}\n{}", tail(&o.output_tail, EVENT_OUTPUT_TAIL))
3280}
3281
3282/// Run configured shell commands in `cwd`, in order.
3283async fn run_commands(
3284    shell: &[String],
3285    commands: &[String],
3286    cwd: &Path,
3287    timeout: Duration,
3288) -> Vec<CommandOutcome> {
3289    let mut out = Vec::new();
3290    for command in commands {
3291        let started = Instant::now();
3292        let mut cmd = tokio::process::Command::new(&shell[0]);
3293        cmd.quiet();
3294        cmd.args(&shell[1..])
3295            .arg(command)
3296            .current_dir(cwd)
3297            .stdin(std::process::Stdio::null())
3298            .stdout(std::process::Stdio::piped())
3299            .stderr(std::process::Stdio::piped())
3300            .kill_on_drop(true);
3301        let spawned = cmd.spawn();
3302        let (code, body) = match spawned {
3303            Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
3304                Ok(Ok(o)) => {
3305                    let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
3306                    body.push_str(&String::from_utf8_lossy(&o.stderr));
3307                    (o.status.code(), body)
3308                }
3309                Ok(Err(e)) => (None, format!("failed to run: {e}")),
3310                Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
3311            },
3312            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
3313        };
3314        out.push(CommandOutcome {
3315            command: command.clone(),
3316            code,
3317            output_tail: tail(&body, OUTPUT_TAIL),
3318            duration_ms: started.elapsed().as_millis() as u64,
3319        });
3320    }
3321    out
3322}
3323
3324/// The merge commit / pull request body: the task, and — when the winning
3325/// review round was not clean — the findings still open and whatever the
3326/// fixer declined, so `merge = "pr"` hands the reader the same material
3327/// `magi show` does rather than a pull request that reads clean while
3328/// `run.json` disagrees.
3329fn pr_body(state: &RunState, winner: char) -> String {
3330    let mut message = format!(
3331        "Merge magi run {} (candidate {winner})\n\n{}",
3332        state.id, state.instruction
3333    );
3334
3335    let open = state.open_findings();
3336    if !open.is_empty() {
3337        message.push_str("\n\n## Open review findings\n\n");
3338        for f in &open {
3339            message.push_str(&format!("- `{}` [{:?}] {}\n", f.id, f.severity, f.title));
3340        }
3341    }
3342
3343    if let Some(fix) = state.reviews.last().and_then(|r| r.fix.as_ref())
3344        && !fix.rejected.is_empty()
3345    {
3346        message.push_str("\n## Declined by the fixer\n\n");
3347        for r in &fix.rejected {
3348            message.push_str(&format!("- `{}`: {}\n", r.id, r.why));
3349        }
3350    }
3351
3352    message
3353}
3354
3355/// `gh pr create`, returning the PR url.
3356async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
3357    let title = body.lines().next().unwrap_or("magi run").to_owned();
3358    let out = tokio::process::Command::new("gh")
3359        .args([
3360            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
3361        ])
3362        .current_dir(cwd)
3363        .stdin(std::process::Stdio::null())
3364        .output()
3365        .await
3366        .context("spawn gh")?;
3367    if out.status.success() {
3368        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
3369    } else {
3370        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
3371    }
3372}
3373
3374/// Tear a run's worktrees and branches down.
3375pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
3376    let repo = state.repo.clone();
3377    let root = state.worktree_root();
3378    let winner = state.tally.as_ref().map(|t| t.winner);
3379    let mut removed = Vec::new();
3380
3381    for i in 0..state.candidates.len() {
3382        let c = state.candidates[i].clone();
3383        let is_winner = Some(c.label) == winner;
3384        if is_winner && !drop_winner {
3385            continue;
3386        }
3387        if c.worktree.exists() {
3388            git::worktree_remove(&repo, &c.worktree).await.ok();
3389            removed.push(c.worktree.to_string_lossy().into_owned());
3390        }
3391        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
3392            git::branch_delete(&repo, &c.branch).await.ok();
3393            removed.push(c.branch.clone());
3394        }
3395        state.candidates[i].folded = true;
3396    }
3397
3398    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
3399        let path = name.path();
3400        let keep = !drop_winner
3401            && winner.is_some_and(|w| {
3402                path.file_name()
3403                    .is_some_and(|n| n == format!("cand-{w}").as_str())
3404            });
3405        if keep {
3406            continue;
3407        }
3408        git::worktree_remove(&repo, &path).await.ok();
3409        removed.push(path.to_string_lossy().into_owned());
3410    }
3411
3412    if state.enabled_worktree_config && drop_winner {
3413        git::disable_worktree_config(&repo).await.ok();
3414        state.enabled_worktree_config = false;
3415    }
3416    state.save()?;
3417    Ok(removed)
3418}
3419
3420/// Severity of the worst open finding in the last review round, for reporting.
3421pub fn worst_open(state: &RunState) -> Option<Severity> {
3422    state
3423        .reviews
3424        .last()?
3425        .reviews
3426        .iter()
3427        .flat_map(|r| r.findings.iter())
3428        .map(|f| f.severity)
3429        .max()
3430}
3431
3432#[cfg(test)]
3433mod tests {
3434    use super::*;
3435    use std::time::Duration;
3436
3437    // `round_is_clean` is the exact decision this task fixed: a round with a
3438    // seat that never answered must not read the same as a round every seat
3439    // actually reviewed. These are deterministic and process-free by design —
3440    // the equivalent end-to-end check (a real reviewer timing out under a
3441    // live graph run) is a genuine race against wall-clock contention, and a
3442    // spawn slow enough to blow even a generous budget under a loaded test
3443    // run must not turn this specific regression check flaky.
3444
3445    #[test]
3446    fn a_full_panel_that_found_nothing_is_clean() {
3447        assert!(round_is_clean(0, true, 2, 2, IncompleteReviewPolicy::Block));
3448    }
3449
3450    #[test]
3451    fn a_missing_seat_is_never_clean_under_the_default_policy() {
3452        assert!(!round_is_clean(
3453            0,
3454            true,
3455            1,
3456            2,
3457            IncompleteReviewPolicy::Block
3458        ));
3459    }
3460
3461    #[test]
3462    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
3463        assert!(!round_is_clean(1, true, 1, 2, IncompleteReviewPolicy::Warn));
3464    }
3465
3466    #[test]
3467    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
3468        assert!(round_is_clean(0, true, 1, 2, IncompleteReviewPolicy::Warn));
3469    }
3470
3471    #[test]
3472    fn a_full_panel_with_an_open_finding_is_not_clean() {
3473        assert!(!round_is_clean(
3474            1,
3475            true,
3476            2,
3477            2,
3478            IncompleteReviewPolicy::Block
3479        ));
3480    }
3481
3482    #[test]
3483    fn a_full_panel_with_a_red_e2e_is_not_clean() {
3484        assert!(!round_is_clean(
3485            0,
3486            false,
3487            2,
3488            2,
3489            IncompleteReviewPolicy::Block
3490        ));
3491    }
3492
3493    // `review_conclusion` is the exact decision the review hand-off task
3494    // fixed: a round budget spent (or a tree that stopped moving) must not
3495    // collapse into `Blocked` regardless of what verification actually
3496    // said. Deterministic and process-free for the same reason the
3497    // `round_is_clean` family above is.
3498    fn review_round(
3499        clean: bool,
3500        blocking: usize,
3501        answered: usize,
3502        expected: usize,
3503        progressed: bool,
3504        e2e_ok: bool,
3505    ) -> ReviewRound {
3506        ReviewRound {
3507            round: 1,
3508            head: "h".to_owned(),
3509            reviews: Vec::new(),
3510            e2e: vec![CommandOutcome {
3511                command: "test".to_owned(),
3512                code: Some(if e2e_ok { 0 } else { 1 }),
3513                output_tail: String::new(),
3514                duration_ms: 0,
3515            }],
3516            verify_retried: false,
3517            fix: None,
3518            blocking,
3519            answered,
3520            expected,
3521            clean,
3522            progressed,
3523        }
3524    }
3525
3526    #[test]
3527    fn review_conclusion_is_none_when_nothing_has_run() {
3528        assert_eq!(review_conclusion(&[], 3), None);
3529    }
3530
3531    #[test]
3532    fn review_conclusion_is_none_while_rounds_remain() {
3533        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
3534        assert_eq!(review_conclusion(&rounds, 3), None);
3535    }
3536
3537    #[test]
3538    fn review_conclusion_is_gating_once_a_round_is_clean() {
3539        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
3540        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
3541    }
3542
3543    #[test]
3544    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
3545        let rounds = vec![
3546            review_round(false, 1, 2, 2, true, true),
3547            review_round(false, 1, 2, 2, true, true),
3548        ];
3549        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
3550    }
3551
3552    #[test]
3553    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
3554        let rounds = vec![
3555            review_round(false, 1, 2, 2, true, true),
3556            review_round(false, 1, 2, 2, true, false),
3557        ];
3558        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
3559    }
3560
3561    #[test]
3562    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
3563        // Missing input, not a verified tree — never a hand-off candidate.
3564        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
3565        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
3566    }
3567
3568    #[test]
3569    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
3570        let rounds = vec![
3571            review_round(false, 1, 2, 2, false, true),
3572            review_round(false, 1, 2, 2, false, true),
3573        ];
3574        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
3575    }
3576
3577    fn secs(n: u64) -> Duration {
3578        Duration::from_secs(n)
3579    }
3580
3581    /// A throwaway repo with one commit on `main`, for tests that need `merge`
3582    /// to make real (and, if it runs at all, real*ly fail*) git calls.
3583    fn init_repo(dir: &Path) {
3584        let run = |args: &[&str]| {
3585            let out = std::process::Command::new("git")
3586                .args(args)
3587                .current_dir(dir)
3588                .output()
3589                .expect("spawn git");
3590            assert!(
3591                out.status.success(),
3592                "git {args:?} failed: {}",
3593                String::from_utf8_lossy(&out.stderr)
3594            );
3595        };
3596        run(&["init", "-b", "main"]);
3597        run(&["config", "user.name", "magi test"]);
3598        run(&["config", "user.email", "magi@example.com"]);
3599        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
3600        run(&["add", "-A"]);
3601        run(&["commit", "-m", "init"]);
3602    }
3603
3604    /// `status == Ready` used to be read as "this is the harmless
3605    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
3606    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
3607    /// run's PR was closed without merging — and reentering `merge` with
3608    /// `mode` still `Pr` does not know the difference, so it pushed and
3609    /// opened a second pull request. `mode == Local` reproduces the same
3610    /// blind spot without a network call: reentry must not attempt another
3611    /// git merge once this node has already recorded an outcome.
3612    #[tokio::test]
3613    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
3614        let tmp = tempfile::tempdir().expect("tempdir");
3615        let repo = tmp.path().join("repo");
3616        std::fs::create_dir_all(&repo).unwrap();
3617        init_repo(&repo);
3618
3619        let mut config = Config::default();
3620        config.merge.mode = MergeMode::Local;
3621
3622        let mut state = RunState::new(
3623            repo.clone(),
3624            "main".to_owned(),
3625            "deadbeef".to_owned(),
3626            "task".to_owned(),
3627            config,
3628        );
3629        state.candidates = vec![Candidate {
3630            index: 0,
3631            label: 'A',
3632            agent: "alpha".to_owned(),
3633            branch: "does-not-exist".to_owned(),
3634            worktree: repo.clone(),
3635            summary: String::new(),
3636            stat: String::new(),
3637            files: 0,
3638            commits: 0,
3639            empty: false,
3640            failed: None,
3641            duration_ms: 0,
3642            folded: false,
3643        }];
3644        state.tally = Some(Tally {
3645            first_choice: BTreeMap::from([('A', 1)]),
3646            borda: BTreeMap::new(),
3647            winner: 'A',
3648            rankings: 1,
3649            unanimous_initial: true,
3650            deliberated: false,
3651            changed_votes: 0,
3652            unanimous_final: true,
3653            tie_break: None,
3654            judges: 0,
3655            present: 0,
3656            quorum: 0,
3657            met_quorum: true,
3658            uncontested: Some("only candidate A produced a change".to_owned()),
3659        });
3660        state.reviews = vec![ReviewRound {
3661            round: 1,
3662            head: "deadbeef".to_owned(),
3663            reviews: Vec::new(),
3664            e2e: Vec::new(),
3665            fix: None,
3666            blocking: 0,
3667            answered: 0,
3668            expected: 0,
3669            clean: true,
3670            verify_retried: false,
3671            progressed: false,
3672        }];
3673        state.gate = vec![CommandOutcome {
3674            command: "test".to_owned(),
3675            code: Some(0),
3676            output_tail: String::new(),
3677            duration_ms: 0,
3678        }];
3679        // Reached its conclusion already — e.g. `land` closing the PR without
3680        // merging it, which (like the honest `MergeMode::None` path) leaves
3681        // `status` at `Ready`. The recorded outcome is what actually marks
3682        // this node done.
3683        state.status = RunStatus::Ready;
3684        state.merge = Some(MergeOutcome {
3685            mode: MergeMode::Local,
3686            ok: false,
3687            detail: "already concluded".to_owned(),
3688        });
3689
3690        let mut runner = Runner {
3691            state,
3692            roles: ResolvedRoles {
3693                implementers: Vec::new(),
3694                judges: Vec::new(),
3695                reviewers: Vec::new(),
3696                fixer: None,
3697            },
3698            sem: Arc::new(Semaphore::new(1)),
3699            pause: Pause::new(),
3700        };
3701
3702        runner.merge().await.expect("merge");
3703
3704        assert_eq!(
3705            runner.state.status,
3706            RunStatus::Ready,
3707            "a concluded run's status must not change on reentry"
3708        );
3709        assert_eq!(
3710            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
3711            Some("already concluded"),
3712            "merge must not run again once the node already recorded an outcome"
3713        );
3714    }
3715
3716    fn state_with_round(round: ReviewRound) -> RunState {
3717        let mut s = RunState::new(
3718            PathBuf::from("/repo"),
3719            "main".to_owned(),
3720            "abc1234".to_owned(),
3721            "add retries".to_owned(),
3722            Config::default(),
3723        );
3724        s.reviews = vec![round];
3725        s
3726    }
3727
3728    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
3729        crate::verdict::Finding {
3730            id: id.to_owned(),
3731            severity,
3732            file: None,
3733            line: None,
3734            title: title.to_owned(),
3735            detail: String::new(),
3736        }
3737    }
3738
3739    #[test]
3740    fn pr_body_names_open_findings_and_declined_ones() {
3741        let round = ReviewRound {
3742            round: 2,
3743            head: "deadbee".to_owned(),
3744            reviews: vec![ReviewRecord {
3745                reviewer: 1,
3746                agent: "alpha".to_owned(),
3747                summary: String::new(),
3748                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
3749                failed: None,
3750                duration_ms: 0,
3751            }],
3752            e2e: vec![CommandOutcome {
3753                command: "cargo test".to_owned(),
3754                code: Some(0),
3755                output_tail: String::new(),
3756                duration_ms: 0,
3757            }],
3758            verify_retried: false,
3759            fix: Some(FixRecord {
3760                agent: "alpha".to_owned(),
3761                addressed: Vec::new(),
3762                rejected: vec![crate::verdict::Rejection {
3763                    id: "R1-1-1".to_owned(),
3764                    why: "not reachable from any caller".to_owned(),
3765                }],
3766                notes: String::new(),
3767                committed: true,
3768                failed: None,
3769                duration_ms: 0,
3770            }),
3771            blocking: 0,
3772            answered: 1,
3773            expected: 1,
3774            clean: false,
3775            progressed: true,
3776        };
3777        let state = state_with_round(round);
3778        let body = pr_body(&state, 'A');
3779
3780        assert!(body.contains("add retries"), "the task must still be there");
3781        assert!(body.contains("R2-1-1"), "{body}");
3782        assert!(body.contains("unused import"), "{body}");
3783        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
3784        assert!(
3785            body.contains("not reachable from any caller"),
3786            "the reason it was declined: {body}"
3787        );
3788    }
3789
3790    #[test]
3791    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
3792        let round = ReviewRound {
3793            round: 1,
3794            head: "deadbee".to_owned(),
3795            reviews: vec![ReviewRecord {
3796                reviewer: 1,
3797                agent: "alpha".to_owned(),
3798                summary: String::new(),
3799                findings: Vec::new(),
3800                failed: None,
3801                duration_ms: 0,
3802            }],
3803            e2e: Vec::new(),
3804            verify_retried: false,
3805            fix: None,
3806            blocking: 0,
3807            answered: 1,
3808            expected: 1,
3809            clean: true,
3810            progressed: false,
3811        };
3812        let state = state_with_round(round);
3813        let body = pr_body(&state, 'A');
3814        assert!(!body.contains("Open review findings"), "{body}");
3815        assert!(!body.contains("Declined"), "{body}");
3816    }
3817
3818    #[test]
3819    fn a_nudge_gets_a_quarter_of_the_budget() {
3820        // The judge and implement budgets magi ships with.
3821        assert_eq!(retry_budget(secs(1200), true), secs(300));
3822        assert_eq!(retry_budget(secs(3600), true), secs(900));
3823    }
3824
3825    #[test]
3826    fn a_resent_prompt_keeps_the_whole_budget() {
3827        // The seat kept no context, so the retry is the original job again and
3828        // shortening it would only guarantee a second failure.
3829        assert_eq!(retry_budget(secs(1200), false), secs(1200));
3830        assert_eq!(retry_budget(secs(60), false), secs(60));
3831    }
3832
3833    #[test]
3834    fn the_floor_never_exceeds_the_original_budget() {
3835        // A short configured timeout must not be *raised* by the floor: the
3836        // operator asked for a bound, and a retry may not outlast the attempt
3837        // it is retrying.
3838        assert_eq!(retry_budget(secs(60), true), secs(60));
3839        assert_eq!(retry_budget(secs(480), true), secs(120));
3840        assert_eq!(retry_budget(secs(0), true), secs(0));
3841    }
3842}