Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use std::sync::atomic::{AtomicBool, Ordering};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::agent::{self, AgentOutput, Invocation, SeatState};
30use crate::blind;
31use crate::config::{
32    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, Prompts, ResolvedRoles,
33};
34use crate::git;
35use crate::land;
36use crate::proc::Quiet as _;
37use crate::prompt::{self, CandidateView, Turn};
38use crate::run::{
39    BaseSync, Candidate, CommandOutcome, DeliberationRound, DeliberationTurn, FixRecord, Judgement,
40    MergeOutcome, QuotaLoss, ReviewRecord, ReviewRound, RunState, RunStatus, Tally, VoteRecord,
41    tail, write_artifact,
42};
43use crate::verdict::{self, FinalVote, FixReport, Position, Ranking, Review, Severity};
44
45/// How much verification output is kept and fed back to the fixer.
46const OUTPUT_TAIL: usize = 8_000;
47
48/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
49/// a base that moved before giving up and leaving the run `Blocked` for a
50/// person.
51///
52/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
53/// that keeps moving faster than a run can catch it is not something more
54/// rebasing fixes, it is a person's call. Not the same *number as*
55/// `land_rounds` - this budget is spent before a pull request exists, land's
56/// after - but bounded for the identical reason, so it uses the same
57/// default. Counted across both call sites in [`Runner::finish_after_tally`]
58/// (once before review, once before the gate), because either one finding
59/// the base still moving is the same signal.
60const BASE_SYNC_ROUNDS: usize = 4;
61
62/// One queued agent invocation.
63///
64/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
65/// CLI hung up on its own stream is asked again from the same job rather than
66/// rebuilt from scratch. See [`Runner::resume_undelivered`].
67#[derive(Clone)]
68struct SeatJob {
69    spec: AgentSpec,
70    seat: SeatState,
71    cwd: PathBuf,
72    prompt: String,
73    timeout: Duration,
74    allow_write: bool,
75    sessions: bool,
76    artifacts: PathBuf,
77    stem: String,
78}
79
80/// How the graph reads one agent invocation.
81///
82/// Quota is split out from an ordinary failure on purpose: a rate-limited call
83/// is known to fail again if retried now, so the retry loop must not spend an
84/// attempt on it. `Dropped` is split out for the opposite reason: unlike
85/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
86/// error JSON, never the agent's answer — a caller that matched only
87/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
88/// left to read that JSON as if it were usable output. `resume_undelivered`
89/// is the only caller that acts on it; everywhere else it is reported like an
90/// ordinary failure.
91enum AgentOutcome {
92    /// A usable output.
93    Ok(AgentOutput),
94    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
95    Quota(AgentOutput),
96    /// The CLI hung up on its own stream after billed work. See
97    /// [`agent::AgentOutput::work_undelivered`].
98    Dropped(AgentOutput),
99    /// Any other failure: a timeout, a bad exit code, an empty reply.
100    Failed(String),
101}
102
103/// A request to park the run at its next node boundary.
104///
105/// Cloning is how the request travels: the loop keeps one handle and hands a
106/// clone to each [`Runner`], and every clone points at the same flag. There
107/// is no channel because there is nothing to send - the only message is
108/// "park", it is idempotent, and a flag cannot be missed by a receiver that
109/// was not listening yet.
110///
111/// The boundary is what makes this cheap. Every node writes the run's state
112/// before the next one starts, and every node skips what is already recorded:
113/// `prep` returns early once candidates exist, `implement` asks only the seats
114/// with nothing on disk, `judge` returns early once judgements exist. So a
115/// parked run resumes into exactly the node it stopped before, and no agent
116/// work is thrown away. Killing the process mid-node, by contrast, loses
117/// whatever the seats in flight had not yet written - which for an implement
118/// wave is an hour of paid work.
119#[derive(Debug, Clone, Default)]
120pub struct Pause(Arc<AtomicBool>);
121
122impl Pause {
123    /// A pause nobody has asked for yet.
124    #[must_use]
125    pub fn new() -> Self {
126        Self::default()
127    }
128
129    /// Ask the run to park at its next node boundary. Idempotent.
130    pub fn park(&self) {
131        self.0.store(true, Ordering::SeqCst);
132    }
133
134    /// Has a park been asked for?
135    #[must_use]
136    pub fn parked(&self) -> bool {
137        self.0.load(Ordering::SeqCst)
138    }
139}
140
141/// Drives one run.
142pub struct Runner {
143    /// Run state; public so the CLI can report on it.
144    pub state: RunState,
145    roles: ResolvedRoles,
146    sem: Arc<Semaphore>,
147    /// Set when someone wants the run parked at its next node boundary.
148    pause: Pause,
149}
150
151/// The commit a run branches from: the base branch as the remote has it.
152///
153/// Two failures this replaces. A run used to branch off `HEAD` and so refused
154/// to start on a dirty tree, which made `magi serve` decline every task for as
155/// long as the operator had work in progress - most of the time. Branching off
156/// the *local* base branch fixed that and introduced a worse one: `land` merges
157/// the winner on GitHub, nothing updates the local ref, and the next run
158/// branches off a base missing everything the previous runs landed. Two tasks
159/// in a row from a phone would have had the second silently re-implementing
160/// against stale code and opening a pull request that reverted the first.
161///
162/// Only refs move here - no checkout, no local branch, no merge - so it is safe
163/// with uncommitted work in the tree. A machine with no network still starts:
164/// the fetch may fail and the local tip is used with a warning, because
165/// refusing to run offline is a worse failure than running against a base the
166/// operator can see for themselves.
167///
168/// One function, called by both entry points. Two answers to "where does a run
169/// branch from" is the kind of drift nobody notices until a diff is wrong.
170async fn resolve_base(repo: &Path, base_branch: &str, remote: &str) -> Result<String> {
171    let tracking = format!("{remote}/{base_branch}");
172    let fetched = git::fetch(repo, remote, base_branch).await;
173    if let Ok(out) = &fetched
174        && out.ok()
175        && git::rev_exists(repo, &tracking).await
176    {
177        return git::rev_parse(repo, &tracking).await;
178    }
179    let why = match &fetched {
180        Ok(out) if !out.ok() => out.stderr.lines().next().unwrap_or("").to_owned(),
181        Ok(_) => format!("{remote} has no {base_branch}"),
182        Err(e) => e.to_string(),
183    };
184    tracing::warn!(
185        "could not read {tracking} ({why}); branching off the local \
186         {base_branch} instead, which may be behind"
187    );
188    git::rev_parse(repo, base_branch).await.with_context(|| {
189        format!(
190            "cannot resolve `{base_branch}`; set [merge] base in magi.toml to a \
191             branch that exists"
192        )
193    })
194}
195
196impl Runner {
197    /// Start a fresh run against `repo`.
198    pub async fn start(repo: &Path, instruction: String, config: Config) -> Result<Self> {
199        let repo = git::toplevel(repo).await?;
200        let missing = agent::missing_programs(&config.agents);
201        if !missing.is_empty() {
202            bail!(
203                "these agent programs are not on PATH: {}. Fix the roster in \
204                 magi.toml or install them.",
205                missing.join(", ")
206            );
207        }
208        let base_branch = match config.merge.base.clone() {
209            Some(b) => b,
210            None => git::current_branch(&repo)
211                .await?
212                .context("HEAD is detached; set [merge] base in magi.toml")?,
213        };
214        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
215        // Still worth saying out loud. The operator's uncommitted work is not
216        // part of this run, and someone watching a candidate fail to use a
217        // change they just made deserves to know why.
218        if !git::is_clean(&repo).await? {
219            tracing::warn!(
220                "{} has uncommitted changes; they are not part of this run, \
221                 which branches off {base_branch} ({})",
222                repo.display(),
223                &base_commit[..base_commit.len().min(8)]
224            );
225        }
226        let roles = config.resolve_roles()?;
227        let max_parallel = config.graph.max_parallel.max(1);
228        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
229        state.event("start", format!("run {} created", state.id));
230        state.save()?;
231        Ok(Self {
232            state,
233            roles,
234            sem: Arc::new(Semaphore::new(max_parallel)),
235            pause: Pause::new(),
236        })
237    }
238
239    /// Open a review-only run against work that already exists on `branch`.
240    ///
241    /// The expensive half of the graph is the implement wave — measured at
242    /// 111 and 134 internal tool-loop turns on this repository, against a
243    /// handful for a judge or a reviewer. The cheap half is worth running on
244    /// hand-written work too, and there was no way to reach it.
245    ///
246    /// No new state and no schema change are needed: a run with **one** viable
247    /// candidate and a tally already decided degrades `execute` to exactly
248    /// review → gate → merge, because `judge` skips a single-candidate field,
249    /// `deliberate` has fewer than two first choices to reconcile, `vote`
250    /// returns early, `tally` is already present and `fold_losers` has no
251    /// losers. Resuming such a run therefore does the right thing as well.
252    pub async fn review(repo: &Path, branch: &str, config: Config) -> Result<Self> {
253        let repo = git::toplevel(repo).await?;
254        let missing = agent::missing_programs(&config.agents);
255        if !missing.is_empty() {
256            bail!(
257                "these agent programs are not on PATH: {}. Fix the roster in \
258                 magi.toml or install them.",
259                missing.join(", ")
260            );
261        }
262        if !git::branch_exists(&repo, branch).await? {
263            bail!("no branch `{branch}` in {}", repo.display());
264        }
265        let base_branch = match config.merge.base.clone() {
266            Some(b) => b,
267            None => git::current_branch(&repo)
268                .await?
269                .context("HEAD is detached; set [merge] base in magi.toml")?,
270        };
271        if base_branch == branch {
272            bail!("`{branch}` is the base branch; there is nothing to review against");
273        }
274        let base_commit = resolve_base(&repo, &base_branch, &config.merge.remote).await?;
275
276        let roles = config.resolve_roles()?;
277        let max_parallel = config.graph.max_parallel.max(1);
278        // The commit subjects are the closest thing to a task statement that
279        // existing work carries, and the reviewers are told as much.
280        let log = git::log_oneline(&repo, &base_commit, branch)
281            .await
282            .unwrap_or_default();
283        let instruction = format!(
284            "Review the work already on branch `{branch}`. There is no task \
285             statement: what the change claims to do is whatever its commits \
286             say.\n\n{}",
287            if log.trim().is_empty() {
288                "(no commit messages)"
289            } else {
290                log.trim()
291            }
292        );
293        let mut state = RunState::new(
294            repo.clone(),
295            base_branch,
296            base_commit.clone(),
297            instruction,
298            config,
299        );
300
301        // An attached worktree, so the fixer's commits land on the branch under
302        // review rather than on a detached head nobody will look at again.
303        let worktree = state.worktree_root().join("under-review");
304        if let Some(parent) = worktree.parent() {
305            tokio::fs::create_dir_all(parent).await.ok();
306        }
307        let path = worktree.to_string_lossy().to_string();
308        git::git(&repo, &["worktree", "add", &path, branch])
309            .await
310            .with_context(|| {
311                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
312            })?;
313
314        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
315            .await
316            .unwrap_or(0);
317        if commits == 0 {
318            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
319        }
320        let files = git::changed_files(&worktree, &base_commit, "HEAD")
321            .await
322            .map(|f| f.len())
323            .unwrap_or(0);
324        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
325            .await
326            .unwrap_or_default();
327
328        state.candidates.push(Candidate {
329            index: 0,
330            label: 'A',
331            // Not an agent id on purpose: nothing in the roster wrote this, and
332            // the stats tables must not credit anyone with a win for it.
333            agent: "(existing branch)".to_owned(),
334            branch: branch.to_owned(),
335            worktree,
336            summary: String::new(),
337            stat,
338            files,
339            commits,
340            empty: false,
341            failed: None,
342            duration_ms: 0,
343            folded: false,
344        });
345        state.tally = Some(Tally {
346            first_choice: BTreeMap::from([('A', 0)]),
347            borda: BTreeMap::new(),
348            winner: 'A',
349            rankings: 0,
350            unanimous_initial: false,
351            deliberated: false,
352            changed_votes: 0,
353            unanimous_final: false,
354            tie_break: None,
355            // No panel sat, so no quorum applies. Zero judges is the correct
356            // number for work that never competed, and must not be reported as
357            // a collapsed panel.
358            judges: 0,
359            present: 0,
360            quorum: 0,
361            met_quorum: true,
362            uncontested: Some("review-only run: nothing competed".to_owned()),
363        });
364        state.status = RunStatus::Reviewing;
365        state.event(
366            "start",
367            format!(
368                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
369                state.id
370            ),
371        );
372        state.save()?;
373        Ok(Self {
374            state,
375            roles,
376            sem: Arc::new(Semaphore::new(max_parallel)),
377            pause: Pause::new(),
378        })
379    }
380
381    /// Reopen an existing run.
382    pub fn resume(id: &str) -> Result<Self> {
383        let state = RunState::load(id)?;
384        let roles = state.config.resolve_roles()?;
385        let max_parallel = state.config.graph.max_parallel.max(1);
386        Ok(Self {
387            state,
388            roles,
389            sem: Arc::new(Semaphore::new(max_parallel)),
390            pause: Pause::new(),
391        })
392    }
393
394    /// Walk the graph to a terminal state, skipping nodes already recorded.
395    pub async fn execute(&mut self) -> Result<()> {
396        // Moving again, so it is no longer parked. Set before the walk rather
397        // than in `resume`, so every way of re-entering the graph clears it
398        // and a card cannot claim a run is waiting to be resumed while the
399        // agents are already working.
400        self.state.parked = false;
401        // Any seat this state still lists as answering belongs to whatever
402        // process last drove this run — this one included, if it crashed
403        // mid-wave. Cleared and flushed immediately, before anything else
404        // runs, so a resume can never show a seat as live when nothing is
405        // asking it anything yet; the node that actually dispatches the next
406        // wave repopulates it.
407        if self.state.clear_active() {
408            self.state.save()?;
409        }
410        // A run that already lost its quorum never resumes into the verdict
411        // machinery: `deliberate` and `vote` would otherwise clobber the
412        // stalled marker back to Voting and the run would keep going past a
413        // verdict that is no longer trustworthy. Everything already recorded is
414        // kept, so the run stays resumable (or foldable) for a human to pick up.
415        //
416        // On --resume the run gets one chance to repair itself: the seats a
417        // rate limit took out are re-asked. If their quota has since reset and
418        // the quorum is restored, the run picks up and finishes; otherwise it
419        // stays stale and still-resumable for a later retry. If it does not
420        // recover, the returned status stays `Stalled` and nothing was
421        // clobbered (the recovery only mutates entries for the lost seats).
422        if self.state.status == RunStatus::Stalled {
423            if self.recover_stall().await? {
424                self.finish_after_tally().await?;
425            } else {
426                // Still below quorum: persist the marker and stay resumable.
427                self.state.save()?;
428            }
429            return Ok(());
430        }
431        self.prep().await?;
432        if self.park_here()? {
433            return Ok(());
434        }
435        self.implement().await?;
436        if self.park_here()? {
437            return Ok(());
438        }
439        self.judge().await?;
440        if self.park_here()? {
441            return Ok(());
442        }
443        self.deliberate().await?;
444        if self.park_here()? {
445            return Ok(());
446        }
447        self.vote().await?;
448        if self.park_here()? {
449            return Ok(());
450        }
451        self.tally()?;
452        // A verdict that lost its quorum is not trustworthy: do not review,
453        // gate, or merge on it. Everything already done is kept, so the run
454        // stays resumable (or foldable); the human can replace the agent that
455        // ran out of quota and pick it up.
456        if self.state.status == RunStatus::Stalled {
457            // Persist the stalled marker now — the normal end-of-execute save
458            // below is below this early return, and without it a resumed run
459            // would reload a pre-tally status and keep going.
460            self.state.save()?;
461            return Ok(());
462        }
463        self.finish_after_tally().await?;
464        Ok(())
465    }
466
467    /// Park here if asked to, recording it in the run's own timeline.
468    ///
469    /// Returns whether the caller should stop walking the graph. The state is
470    /// saved either way by the node that just finished; this adds the event so
471    /// the operator's card says why a run that is neither finished nor moving
472    /// is sitting where it is.
473    fn park_here(&mut self) -> Result<bool> {
474        if !self.pause.parked() {
475            return Ok(false);
476        }
477        self.state.event(
478            "park",
479            format!(
480                "parked after `{}` — resume to carry on from here",
481                self.state.status.as_str()
482            ),
483        );
484        self.state.parked = true;
485        self.state.save()?;
486        Ok(true)
487    }
488
489    /// Hand the runner a pause to watch.
490    pub fn on_pause(&mut self, pause: Pause) {
491        self.pause = pause;
492    }
493
494    /// The tail of the graph after a trustworthy tally: fold losers, review,
495    /// gate, merge, and persist.
496    async fn finish_after_tally(&mut self) -> Result<()> {
497        self.fold_losers().await?;
498        // Before review starts, and again right before the gate: a run's
499        // review rounds can themselves take long enough for the base to move
500        // a second time, and the gate is the one node whose "green" gets
501        // acted on.
502        self.sync_to_base().await?;
503        self.review_loop().await?;
504        self.sync_to_base().await?;
505        self.gate().await?;
506        self.merge().await?;
507        self.state.save()?;
508        Ok(())
509    }
510
511    // ---------------------------------------------------------------- prep
512
513    async fn prep(&mut self) -> Result<()> {
514        if !self.state.candidates.is_empty() {
515            return Ok(());
516        }
517        self.state.status = RunStatus::Prep;
518        let repo = self.state.repo.clone();
519        let base = self.state.base_commit.clone();
520        let root = self.state.worktree_root();
521        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
522
523        // The hook is the write-time half of the blindness contract; the
524        // presentation filter in `blind` is the half that cannot be bypassed.
525        let hooks_dir = self.state.dir().join("hooks");
526        if self.state.config.blind.commit_msg_hook {
527            std::fs::create_dir_all(&hooks_dir)
528                .with_context(|| format!("create {}", hooks_dir.display()))?;
529            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
530            let path = hooks_dir.join("commit-msg");
531            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
532            make_executable(&path)?;
533            if git::enable_worktree_config(&repo).await? {
534                self.state.enabled_worktree_config = true;
535            }
536        }
537
538        for (index, (spec, label)) in self
539            .roles
540            .implementers
541            .clone()
542            .into_iter()
543            .zip(labels)
544            .enumerate()
545        {
546            let branch = self.state.branch_for(label);
547            let worktree = root.join(format!("cand-{label}"));
548            git::worktree_add_branch(&repo, &worktree, &branch, &base).await?;
549            if self.state.config.blind.commit_msg_hook {
550                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
551            }
552            git::local_exclude(&worktree, "/.magi/").await?;
553            self.state.candidates.push(Candidate {
554                index,
555                label,
556                agent: spec.id.clone(),
557                branch,
558                worktree,
559                summary: String::new(),
560                stat: String::new(),
561                files: 0,
562                commits: 0,
563                empty: false,
564                failed: None,
565                duration_ms: 0,
566                folded: false,
567            });
568        }
569
570        for j in 1..=self.roles.judges.len() {
571            let wt = root.join(format!("judge-{j}"));
572            if !wt.exists() {
573                git::worktree_add_detached(&repo, &wt, &base).await?;
574            }
575        }
576
577        // A judge cannot tell it is looking at its own patch — the seats keep
578        // separate conversations — but a panel that shares agents with the
579        // field is less independent than it looks, and that is worth saying out
580        // loud once per run rather than leaving it in the config.
581        let authors: Vec<&str> = self
582            .roles
583            .implementers
584            .iter()
585            .map(|a| a.id.as_str())
586            .collect();
587        let overlap: Vec<String> = self
588            .roles
589            .judges
590            .iter()
591            .enumerate()
592            .filter(|(_, j)| authors.contains(&j.id.as_str()))
593            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
594            .collect();
595        if !overlap.is_empty() {
596            let note = format!(
597                "{} also authored a candidate; blind, but the panel is less \
598                 independent than {} distinct agents would be",
599                overlap.join(", "),
600                self.roles.judges.len()
601            );
602            self.state.event("prep", note);
603        }
604
605        self.state.event(
606            "prep",
607            format!(
608                "{} candidates, {} judges, base {} ({})",
609                self.state.candidates.len(),
610                self.roles.judges.len(),
611                &self.state.base_commit[..7.min(self.state.base_commit.len())],
612                self.state.base_branch
613            ),
614        );
615        self.state.status = RunStatus::Implementing;
616        self.state.save()?;
617        Ok(())
618    }
619
620    // ----------------------------------------------------------- implement
621
622    async fn implement(&mut self) -> Result<()> {
623        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
624        // agent files with `magi task add` name the run that paid for it. The
625        // prompt overlay is cloned alongside it because the waves borrow it
626        // while `self` is mutably borrowed by the node's own bookkeeping.
627        let run_id = self.state.id.clone();
628        let prompts = self.state.config.prompts.clone();
629        let todo: Vec<usize> = self
630            .state
631            .candidates
632            .iter()
633            .enumerate()
634            .filter(|(_, c)| c.commits == 0 && c.failed.is_none() && !c.empty)
635            .map(|(i, _)| i)
636            .collect();
637        if todo.is_empty() {
638            return self.after_implement();
639        }
640        self.state.status = RunStatus::Implementing;
641
642        let language = self.state.config.graph.language.clone();
643        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
644        let sessions = self.state.config.graph.sessions;
645        let artifacts = agent::artifacts_dir(&self.state.dir());
646
647        let mut jobs = Vec::new();
648        for &i in &todo {
649            let (index, label, worktree) = {
650                let c = &self.state.candidates[i];
651                (c.index, c.label, c.worktree.clone())
652            };
653            let spec = self.roles.implementers[index].clone();
654            let seat_key = format!("impl-{label}");
655            let seat = self.seat(&seat_key, &spec.id);
656            let instruction = self.state.instruction.clone();
657            jobs.push(SeatJob {
658                spec,
659                seat,
660                prompt: prompt::implement(&instruction, &worktree.to_string_lossy(), &language),
661                cwd: worktree,
662                timeout,
663                allow_write: true,
664                sessions,
665                artifacts: artifacts.clone(),
666                stem: format!("impl-{label}"),
667            });
668        }
669
670        self.state.event(
671            "implement",
672            format!("{} candidates in parallel", jobs.len()),
673        );
674        // Kept so a seat whose CLI hung up can be asked again from the same
675        // job: `wave` consumes what it is given.
676        let sent = jobs.clone();
677        let cache = self.state.config.cache_dir();
678        let ctx = WaveCtx {
679            run: &run_id,
680            node: "implement",
681            prompts: &prompts,
682            cache: cache.as_deref(),
683        };
684        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
685        self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
686            .await;
687
688        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
689            let seat_key = seat.key.clone();
690            self.state.seats.insert(seat.key.clone(), seat);
691            let label = self.state.candidates[i].label;
692            let worktree = self.state.candidates[i].worktree.clone();
693            let base = self.state.base_commit.clone();
694
695            let (summary, duration, failed) = match out {
696                AgentOutcome::Ok(o) => {
697                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
698                    let failed = (!o.usable()).then(|| {
699                        if o.timed_out {
700                            "agent timed out".to_owned()
701                        } else {
702                            format!("agent exited with {:?}", o.exit_code)
703                        }
704                    });
705                    (text, o.duration_ms, failed)
706                }
707                // Left un-resumed by `resume_undelivered` (a dirty tree
708                // already rescues the work, or there was no session left to
709                // resume into) — reported like the ordinary failure it is,
710                // never as if `o.text` (the CLI's raw error JSON) were an
711                // answer.
712                AgentOutcome::Dropped(o) => {
713                    let why = o
714                        .dropped
715                        .as_ref()
716                        .map(|d| d.why.as_str())
717                        .unwrap_or("the CLI ended the stream without delivering its answer");
718                    (
719                        String::new(),
720                        o.duration_ms,
721                        Some(format!("the CLI dropped the stream ({why})")),
722                    )
723                }
724                AgentOutcome::Quota(o) => {
725                    self.state.quota.push(QuotaLoss {
726                        seat: seat_key,
727                        node: "implement".to_owned(),
728                        at: Timestamp::now(),
729                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
730                    });
731                    (
732                        String::new(),
733                        o.duration_ms,
734                        Some("rate limited (quota); produced no change".to_owned()),
735                    )
736                }
737                AgentOutcome::Failed(e) => (String::new(), 0, Some(e)),
738            };
739
740            // Rescue anything the agent edited but never committed: an
741            // uncommitted candidate would silently be an empty one.
742            let rescued = git::commit_all(
743                &worktree,
744                &format!("magi: candidate {label} (uncommitted work)"),
745            )
746            .await
747            .unwrap_or(false);
748            let commits = git::commits_ahead(&worktree, &base, "HEAD")
749                .await
750                .unwrap_or(0);
751            let patch = git::diff(&worktree, &base, "HEAD")
752                .await
753                .unwrap_or_default();
754            let stat = git::diff_stat(&worktree, &base, "HEAD")
755                .await
756                .unwrap_or_default();
757            let files = git::changed_files(&worktree, &base, "HEAD")
758                .await
759                .map(|f| f.len())
760                .unwrap_or(0);
761            write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
762
763            let c = &mut self.state.candidates[i];
764            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
765            c.stat = stat;
766            c.files = files;
767            c.commits = commits;
768            c.duration_ms = duration;
769            c.empty = commits == 0 || patch.trim().is_empty();
770            // An agent that failed but still produced a committed change stays
771            // in the running: the patch is what gets judged, not the exit code.
772            c.failed = match failed {
773                Some(_) if c.empty => failed,
774                _ => None,
775            };
776            let note = match (&c.failed, c.empty, rescued) {
777                (Some(e), _, _) => format!("candidate {label}: {e}"),
778                (None, true, _) => format!("candidate {label}: no change produced"),
779                (None, false, true) => {
780                    format!(
781                        "candidate {label}: {files} files, {commits} commits (rescued an uncommitted tree)"
782                    )
783                }
784                (None, false, false) => {
785                    format!("candidate {label}: {files} files, {commits} commits")
786                }
787            };
788            self.state.event("implement", note);
789            self.state.save()?;
790        }
791
792        self.after_implement()
793    }
794
795    /// Ask again, once, for work a CLI did and then failed to hand over.
796    ///
797    /// [`agent::dropped_stream`] recognises the one shape observed: an error
798    /// status with an empty response and a usage report showing output tokens,
799    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
800    /// seven minutes and 14,267 output tokens that arrived as an empty
801    /// candidate, because `agy`'s own subscriber fell behind and hung up.
802    ///
803    /// Two conditions, and both matter:
804    ///
805    /// - **Only when the tree is untouched.** Often the agent has already
806    ///   written its files and only the closing message was lost; the rescue
807    ///   commit below picks that up and there is nothing to ask for. Re-asking
808    ///   then would pay for a second implementation of work already on disk.
809    /// - **Once.** A CLI that drops one stream can drop the next, and this
810    ///   node is the most expensive in the graph.
811    ///
812    /// The re-ask is a resume, not a re-run: `has_context` is true because the
813    /// dropped reply still carried its `conversation_id`, so the seat is asked
814    /// to finish what it was doing rather than sent the whole task again. It
815    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
816    /// node's - for the same reason a re-ranked judge does: restating finished
817    /// work is not the work.
818    ///
819    /// Unlike a quota this is worth retrying at all: a rate limit fails the
820    /// same way until it resets, while an abandoned conversation is still
821    /// there to be picked up.
822    async fn resume_undelivered(
823        &mut self,
824        results: &mut [(usize, SeatState, AgentOutcome)],
825        sent: &[SeatJob],
826        prompts: &Prompts,
827        run_id: &str,
828    ) {
829        for (wi, seat, out) in results.iter_mut() {
830            let Some(dropped) = (match &*out {
831                AgentOutcome::Dropped(o) => o.dropped.clone(),
832                _ => None,
833            }) else {
834                continue;
835            };
836            let Some(job) = sent.get(*wi) else { continue };
837            // Already on disk? Then only the closing message was lost.
838            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
839                self.state.event(
840                    "implement",
841                    format!(
842                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
843                         work is in the tree",
844                        seat.key, dropped.output_tokens, dropped.why
845                    ),
846                );
847                continue;
848            }
849            // The re-ask only makes sense as a resume: `resume_after_drop`
850            // says nothing about the task, trusting the seat to still hold it.
851            // Without a session to resume — sessions disabled, or this CLI's
852            // drop shape happened not to carry a session id — that prompt
853            // would open a brand-new conversation with no context at all,
854            // which is worse than leaving this as the ordinary failure it
855            // already is.
856            if !has_context(&job.spec, seat, job.sessions) {
857                self.state.event(
858                    "implement",
859                    format!(
860                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
861                         is no session left to resume",
862                        seat.key, dropped.output_tokens, dropped.why
863                    ),
864                );
865                continue;
866            }
867            self.state.event(
868                "implement",
869                format!(
870                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
871                     conversation",
872                    seat.key, dropped.output_tokens, dropped.why
873                ),
874            );
875            let mut retry = job.clone();
876            retry.seat = seat.clone();
877            retry.prompt = prompt::resume_after_drop(&dropped.why);
878            retry.timeout = retry_budget(job.timeout, true);
879            retry.stem = format!("{}-resume", job.stem);
880            let cache = self.state.config.cache_dir();
881            let ctx = WaveCtx {
882                run: run_id,
883                node: "implement",
884                prompts,
885                cache: cache.as_deref(),
886            };
887            let (resumed_seat, resumed) =
888                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
889            *seat = resumed_seat;
890            *out = resumed;
891        }
892    }
893
894    fn after_implement(&mut self) -> Result<()> {
895        // Scan every candidate patch once the set is complete.
896        if self.state.leaks.is_empty() {
897            let cfg = self.state.config.blind.clone();
898            let mut leaks = Vec::new();
899            for c in &self.state.candidates {
900                let Some(patch) =
901                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
902                else {
903                    continue;
904                };
905                leaks.extend(blind::scan(
906                    &format!("candidate {} patch", c.label),
907                    &patch,
908                    &cfg.vendor_tokens,
909                ));
910            }
911            if !leaks.is_empty() {
912                let summary = leaks
913                    .iter()
914                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
915                    .collect::<Vec<_>>()
916                    .join(", ");
917                match cfg.on_leak {
918                    LeakPolicy::Fail => {
919                        self.state.status = RunStatus::Failed;
920                        self.state
921                            .event("blind", format!("vendor text in a patch: {summary}"));
922                        self.state.leaks = leaks;
923                        self.state.save()?;
924                        bail!(
925                            "blind.on_leak = \"fail\" and vendor text reached a \
926                             judged patch: {summary}"
927                        );
928                    }
929                    LeakPolicy::Redact => self.state.event(
930                        "blind",
931                        format!("redacting vendor text for judging: {summary}"),
932                    ),
933                    LeakPolicy::Warn => self.state.event(
934                        "blind",
935                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
936                    ),
937                }
938                self.state.leaks = leaks;
939            }
940        }
941
942        if self.state.viable().is_empty() {
943            self.state.status = RunStatus::Failed;
944            self.state.save()?;
945            bail!("no candidate produced a change; nothing to judge");
946        }
947        self.state.status = RunStatus::Judging;
948        self.state.save()?;
949        Ok(())
950    }
951
952    // --------------------------------------------------------------- judge
953
954    async fn judge(&mut self) -> Result<()> {
955        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
956        // agent files with `magi task add` name the run that paid for it. The
957        // prompt overlay is cloned alongside it because the waves borrow it
958        // while `self` is mutably borrowed by the node's own bookkeeping.
959        let run_id = self.state.id.clone();
960        let prompts = self.state.config.prompts.clone();
961        if !self.state.judgements.is_empty() || self.state.judge_skipped {
962            return Ok(());
963        }
964        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
965        if viable.len() == 1 {
966            // Recorded so this is a one-time event: `judgements` stays empty
967            // either way, which without this flag is indistinguishable from
968            // "not yet judged" on the next reentry — and status is left
969            // untouched, so a later node's conclusion (e.g. `Blocked` after
970            // the review budget ran out) survives a resume instead of being
971            // clobbered back to `Judging` by this node running again.
972            self.state.judge_skipped = true;
973            self.state.event(
974                "judge",
975                format!(
976                    "only candidate {} produced a change; judging skipped",
977                    viable[0].label
978                ),
979            );
980            self.state.save()?;
981            return Ok(());
982        }
983        self.state.status = RunStatus::Judging;
984
985        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
986        let language = self.state.config.graph.language.clone();
987        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
988        let sessions = self.state.config.graph.sessions;
989        let artifacts = agent::artifacts_dir(&self.state.dir());
990        let root = self.state.worktree_root();
991        let base_short = short(&self.state.base_commit);
992
993        let mut jobs = Vec::new();
994        let mut orders = Vec::new();
995        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
996            let order = blind::presentation_order(viable.len(), j, self.state.seed);
997            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
998            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
999            let seat_key = format!("judge-{}", j + 1);
1000            let seat = self.seat(&seat_key, &spec.id);
1001            jobs.push(SeatJob {
1002                prompt: prompt::judge(
1003                    &self.state.instruction,
1004                    &views,
1005                    self.roles.judges.len(),
1006                    &base_short,
1007                    &language,
1008                ),
1009                spec,
1010                seat,
1011                cwd: root.join(format!("judge-{}", j + 1)),
1012                timeout,
1013                allow_write: false,
1014                sessions,
1015                artifacts: artifacts.clone(),
1016                stem: format!("judge-{}", j + 1),
1017            });
1018        }
1019
1020        self.state.event(
1021            "judge",
1022            format!(
1023                "{} judges ranking {} candidates blind",
1024                jobs.len(),
1025                viable.len()
1026            ),
1027        );
1028        let labels_for_check = labels.clone();
1029        let mut quota_losses = Vec::new();
1030        let cache = self.state.config.cache_dir();
1031        let ctx = WaveCtx {
1032            run: &run_id,
1033            node: "judge",
1034            prompts: &prompts,
1035            cache: cache.as_deref(),
1036        };
1037        let results = ask_json_wave::<Ranking>(
1038            jobs,
1039            Arc::clone(&self.sem),
1040            self.state.config.graph.retries,
1041            &ctx,
1042            &mut quota_losses,
1043            &mut self.state,
1044            &move |r: &Ranking| r.validate(&labels_for_check),
1045        )
1046        .await;
1047        self.state.quota.extend(quota_losses);
1048
1049        for (j, (seat, res)) in results.into_iter().enumerate() {
1050            let agent_id = seat.agent.clone();
1051            self.state.seats.insert(seat.key.clone(), seat);
1052            let mut record = Judgement {
1053                judge: j + 1,
1054                seat: format!("judge-{}", j + 1),
1055                agent: agent_id,
1056                ranking: Vec::new(),
1057                reasons: BTreeMap::new(),
1058                confidence: None,
1059                order: orders[j].clone(),
1060                failed: None,
1061                duration_ms: 0,
1062            };
1063            match res {
1064                Ok((ranking, out)) => {
1065                    record.ranking = ranking.normalized();
1066                    record.reasons = ranking.reasons;
1067                    record.confidence = ranking.confidence;
1068                    record.duration_ms = out.duration_ms;
1069                    self.state.event(
1070                        "judge",
1071                        format!(
1072                            "judge {} ranked {}",
1073                            j + 1,
1074                            record.ranking.iter().collect::<String>()
1075                        ),
1076                    );
1077                }
1078                Err(e) => {
1079                    record.failed = Some(e.to_string());
1080                    self.state
1081                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
1082                }
1083            }
1084            self.state.judgements.push(record);
1085            self.state.save()?;
1086        }
1087        Ok(())
1088    }
1089
1090    // ---------------------------------------------------------- deliberate
1091
1092    async fn deliberate(&mut self) -> Result<()> {
1093        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1094        // agent files with `magi task add` name the run that paid for it. The
1095        // prompt overlay is cloned alongside it because the waves borrow it
1096        // while `self` is mutably borrowed by the node's own bookkeeping.
1097        let run_id = self.state.id.clone();
1098        let prompts = self.state.config.prompts.clone();
1099        if !self.state.deliberation.is_empty() {
1100            return Ok(());
1101        }
1102        let tops: Vec<char> = self
1103            .state
1104            .judgements
1105            .iter()
1106            .filter_map(|j| j.ranking.first().copied())
1107            .collect();
1108        let rounds = self.state.config.graph.deliberate_rounds;
1109        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
1110            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
1111                self.state.event(
1112                    "deliberate",
1113                    format!("judges agreed on {} outright; no deliberation", tops[0]),
1114                );
1115            }
1116            self.state.status = RunStatus::Voting;
1117            self.state.save()?;
1118            return Ok(());
1119        }
1120
1121        self.state.status = RunStatus::Deliberating;
1122        self.state.event(
1123            "deliberate",
1124            format!(
1125                "split: first choices were {} — opening {rounds} round(s)",
1126                tops.iter().collect::<String>()
1127            ),
1128        );
1129
1130        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1131        let language = self.state.config.graph.language.clone();
1132        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1133        let sessions = self.state.config.graph.sessions;
1134        let artifacts = agent::artifacts_dir(&self.state.dir());
1135        let root = self.state.worktree_root();
1136        let base_short = short(&self.state.base_commit);
1137
1138        // Judges argue in sequence so that a turn can answer the one before it;
1139        // that is the difference between deliberation and three parallel
1140        // monologues.
1141        for round in 1..=rounds {
1142            let mut turns: Vec<DeliberationTurn> = Vec::new();
1143            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1144                if self.state.judgements[j].failed.is_some() {
1145                    continue;
1146                }
1147                let seat_key = format!("judge-{}", j + 1);
1148                let mut seat = self.seat(&seat_key, &spec.id);
1149                let transcript = self.transcript(&turns, j);
1150                let context = if has_context(&spec, &seat, sessions) {
1151                    None
1152                } else {
1153                    Some(self.candidate_block(&viable, &base_short))
1154                };
1155                let text = prompt::deliberate(
1156                    &self.state.instruction,
1157                    context.as_deref(),
1158                    &transcript,
1159                    round,
1160                    rounds,
1161                    &language,
1162                );
1163                let job = SeatJob {
1164                    spec,
1165                    seat: seat.clone(),
1166                    prompt: text,
1167                    cwd: root.join(format!("judge-{}", j + 1)),
1168                    timeout,
1169                    allow_write: false,
1170                    sessions,
1171                    artifacts: artifacts.clone(),
1172                    stem: format!("delib-{round}-judge-{}", j + 1),
1173                };
1174                let cache = self.state.config.cache_dir();
1175                let ctx = WaveCtx {
1176                    run: &run_id,
1177                    node: "deliberate",
1178                    prompts: &prompts,
1179                    cache: cache.as_deref(),
1180                };
1181                let (updated, out) =
1182                    run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
1183                seat = updated;
1184                let agent_id = seat.agent.clone();
1185                let seat_key = seat.key.clone();
1186                self.state.seats.insert(seat.key.clone(), seat);
1187                let body = match out {
1188                    AgentOutcome::Ok(o) => verdict::section(&o.text, "position").unwrap_or(o.text),
1189                    // Never read the CLI's raw error JSON as this judge's
1190                    // position — skip the seat instead, the same as any other
1191                    // failed turn.
1192                    AgentOutcome::Dropped(o) => {
1193                        let why =
1194                            o.dropped.as_ref().map(|d| d.why.as_str()).unwrap_or(
1195                                "the CLI ended the stream without delivering its answer",
1196                            );
1197                        self.state.event(
1198                            "deliberate",
1199                            format!(
1200                                "judge {} skipped: the CLI dropped the stream ({why})",
1201                                j + 1
1202                            ),
1203                        );
1204                        continue;
1205                    }
1206                    AgentOutcome::Quota(o) => {
1207                        self.state.quota.push(QuotaLoss {
1208                            seat: seat_key,
1209                            node: "deliberate".to_owned(),
1210                            at: Timestamp::now(),
1211                            reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
1212                        });
1213                        self.state.event(
1214                            "deliberate",
1215                            format!("judge {} skipped: rate limited (quota)", j + 1),
1216                        );
1217                        continue;
1218                    }
1219                    AgentOutcome::Failed(e) => {
1220                        self.state
1221                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
1222                        continue;
1223                    }
1224                };
1225                let tentative = verdict::extract_json::<Position>(&body)
1226                    .ok()
1227                    .and_then(|p| p.tentative)
1228                    .and_then(|s| s.trim().chars().next())
1229                    .map(|c| c.to_ascii_uppercase());
1230                self.state.event(
1231                    "deliberate",
1232                    format!(
1233                        "round {round}: judge {} now favours {}",
1234                        j + 1,
1235                        tentative.map_or("—".to_owned(), |c| c.to_string())
1236                    ),
1237                );
1238                turns.push(DeliberationTurn {
1239                    judge: j + 1,
1240                    agent: agent_id,
1241                    body: blind::sanitize_prose(&body, &self.state.config.blind),
1242                    tentative,
1243                });
1244            }
1245            self.state
1246                .deliberation
1247                .push(DeliberationRound { round, turns });
1248            self.state.save()?;
1249        }
1250
1251        self.state.status = RunStatus::Voting;
1252        self.state.save()?;
1253        Ok(())
1254    }
1255
1256    // ---------------------------------------------------------------- vote
1257
1258    async fn vote(&mut self) -> Result<()> {
1259        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1260        // agent files with `magi task add` name the run that paid for it. The
1261        // prompt overlay is cloned alongside it because the waves borrow it
1262        // while `self` is mutably borrowed by the node's own bookkeeping.
1263        let run_id = self.state.id.clone();
1264        let prompts = self.state.config.prompts.clone();
1265        if !self.state.votes.is_empty() {
1266            return Ok(());
1267        }
1268        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1269        if viable.len() == 1 {
1270            return Ok(());
1271        }
1272        self.state.status = RunStatus::Voting;
1273
1274        let language = self.state.config.graph.language.clone();
1275        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1276        let sessions = self.state.config.graph.sessions;
1277        let artifacts = agent::artifacts_dir(&self.state.dir());
1278        let root = self.state.worktree_root();
1279        let base_short = short(&self.state.base_commit);
1280        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1281
1282        let mut jobs = Vec::new();
1283        let mut seats_at = Vec::new();
1284        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
1285            if self
1286                .state
1287                .judgements
1288                .get(j)
1289                .is_some_and(|r| r.failed.is_some())
1290            {
1291                continue;
1292            }
1293            let seat_key = format!("judge-{}", j + 1);
1294            let seat = self.seat(&seat_key, &spec.id);
1295            let mut text = prompt::final_vote(&viable, &language);
1296            if !has_context(&spec, &seat, sessions) {
1297                text = format!(
1298                    "{}\n\n# Candidates\n\n{}",
1299                    text,
1300                    self.candidate_block(&candidates, &base_short)
1301                );
1302            }
1303            jobs.push(SeatJob {
1304                spec,
1305                seat,
1306                prompt: text,
1307                cwd: root.join(format!("judge-{}", j + 1)),
1308                timeout,
1309                allow_write: false,
1310                sessions,
1311                artifacts: artifacts.clone(),
1312                stem: format!("vote-judge-{}", j + 1),
1313            });
1314            seats_at.push(j);
1315        }
1316
1317        self.state.event(
1318            "vote",
1319            format!(
1320                "collecting {} final votes one by one, privately",
1321                jobs.len()
1322            ),
1323        );
1324        let allowed = viable.clone();
1325        let mut quota_losses = Vec::new();
1326        let cache = self.state.config.cache_dir();
1327        let ctx = WaveCtx {
1328            run: &run_id,
1329            node: "vote",
1330            prompts: &prompts,
1331            cache: cache.as_deref(),
1332        };
1333        let results = ask_json_wave::<FinalVote>(
1334            jobs,
1335            Arc::clone(&self.sem),
1336            self.state.config.graph.retries,
1337            &ctx,
1338            &mut quota_losses,
1339            &mut self.state,
1340            &move |v: &FinalVote| match v.label() {
1341                Some(c) if allowed.contains(&c) => Ok(()),
1342                other => bail!("vote {other:?} is not one of {allowed:?}"),
1343            },
1344        )
1345        .await;
1346        self.state.quota.extend(quota_losses);
1347
1348        for (&j, (seat, res)) in seats_at.iter().zip(results) {
1349            let agent_id = seat.agent.clone();
1350            self.state.seats.insert(seat.key.clone(), seat);
1351            let initial = self
1352                .state
1353                .judgements
1354                .get(j)
1355                .and_then(|r| r.ranking.first().copied());
1356            let mut record = VoteRecord {
1357                judge: j + 1,
1358                agent: agent_id,
1359                vote: None,
1360                reason: String::new(),
1361                changed: false,
1362            };
1363            match res {
1364                Ok((v, _)) => {
1365                    record.vote = v.label();
1366                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1367                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
1368                    self.state.event(
1369                        "vote",
1370                        format!(
1371                            "judge {} voted {}{}",
1372                            j + 1,
1373                            record.vote.unwrap_or('?'),
1374                            if record.changed { " (changed)" } else { "" }
1375                        ),
1376                    );
1377                }
1378                Err(e) => {
1379                    self.state
1380                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
1381                }
1382            }
1383            self.state.votes.push(record);
1384            self.state.save()?;
1385        }
1386        Ok(())
1387    }
1388
1389    // --------------------------------------------------------------- tally
1390
1391    fn tally(&mut self) -> Result<()> {
1392        if self.state.tally.is_some() {
1393            return Ok(());
1394        }
1395        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
1396        let tops: Vec<char> = self
1397            .state
1398            .judgements
1399            .iter()
1400            .filter_map(|j| j.ranking.first().copied())
1401            .collect();
1402        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
1403
1404        // A judge whose private vote failed still counted once, in the initial
1405        // ranking; using it beats discarding a whole seat.
1406        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1407        let mut cast: Vec<char> = Vec::new();
1408        for (i, j) in self.state.judgements.iter().enumerate() {
1409            let vote = self
1410                .state
1411                .votes
1412                .iter()
1413                .find(|v| v.judge == i + 1)
1414                .and_then(|v| v.vote)
1415                .or_else(|| j.ranking.first().copied());
1416            if let Some(v) = vote {
1417                *first_choice.entry(v).or_insert(0) += 1;
1418                cast.push(v);
1419            }
1420        }
1421
1422        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
1423        for j in &self.state.judgements {
1424            let n = j.ranking.len();
1425            for (pos, label) in j.ranking.iter().enumerate() {
1426                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
1427            }
1428        }
1429
1430        let best = first_choice.values().copied().max().unwrap_or(0);
1431        let mut leaders: Vec<char> = first_choice
1432            .iter()
1433            .filter(|(_, v)| **v == best)
1434            .map(|(k, _)| *k)
1435            .collect();
1436        let mut tie_break = None;
1437        if leaders.len() > 1 {
1438            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
1439            let borda_leaders: Vec<char> = leaders
1440                .iter()
1441                .copied()
1442                .filter(|l| borda[l] == top_borda)
1443                .collect();
1444            tie_break = Some(if borda_leaders.len() == 1 {
1445                format!(
1446                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
1447                    leaders.len()
1448                )
1449            } else {
1450                format!(
1451                    "{} way tie on both first-choice votes and Borda points, broken by label order",
1452                    leaders.len()
1453                )
1454            });
1455            leaders = borda_leaders;
1456            leaders.sort_unstable();
1457        }
1458        let winner = *leaders
1459            .first()
1460            .or(viable.first())
1461            .context("no candidate to declare a winner from")?;
1462
1463        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
1464        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
1465        let deliberated = !self.state.deliberation.is_empty();
1466
1467        // Whose verdict is this? A rate-limited seat is absent even if it
1468        // ranked before the limit hit, so presence is measured against the
1469        // recorded losses, not just "did a ranking ever appear".
1470        let quota_seats: std::collections::BTreeSet<&str> =
1471            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1472        let mut present = 0usize;
1473        for (i, j) in self.state.judgements.iter().enumerate() {
1474            if quota_seats.contains(j.seat.as_str()) {
1475                continue;
1476            }
1477            let ranked = !j.ranking.is_empty() && j.failed.is_none();
1478            let voted = self
1479                .state
1480                .votes
1481                .iter()
1482                .any(|v| v.judge == i + 1 && v.vote.is_some());
1483            if ranked || voted {
1484                present += 1;
1485            }
1486        }
1487        // Strict majority of the configured panel. A bare majority is real
1488        // signal we can act on, while a minority verdict must never stand in
1489        // for a healthy one. A one-candidate run needs no panel at all, and
1490        // `judges` stays `0` rather than the roster size a panel that never
1491        // sat would otherwise be credited with.
1492        let needs_quorum = viable.len() > 1;
1493        let judges_total = if needs_quorum {
1494            self.roles.judges.len()
1495        } else {
1496            0
1497        };
1498        let quorum = if needs_quorum {
1499            judges_total / 2 + 1
1500        } else {
1501            0
1502        };
1503        let met_quorum = !needs_quorum || present >= quorum;
1504        let uncontested = (!needs_quorum).then(|| {
1505            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
1506        });
1507
1508        self.state.event(
1509            "tally",
1510            match &uncontested {
1511                Some(reason) => format!("winner {winner} — {reason}"),
1512                None => format!(
1513                    "winner {winner} — votes {} | initial {} | {} changed | \
1514                     {present}/{judges_total} judges{}",
1515                    first_choice
1516                        .iter()
1517                        .map(|(k, v)| format!("{k}:{v}"))
1518                        .collect::<Vec<_>>()
1519                        .join(" "),
1520                    if unanimous_initial {
1521                        "unanimous"
1522                    } else {
1523                        "split"
1524                    },
1525                    changed_votes,
1526                    if met_quorum {
1527                        String::new()
1528                    } else {
1529                        format!(" — below quorum ({quorum} required)")
1530                    },
1531                ),
1532            },
1533        );
1534        if !met_quorum {
1535            self.state.event(
1536                "stall",
1537                format!(
1538                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
1539                     the run stops here, resumable"
1540                ),
1541            );
1542        }
1543        self.state.tally = Some(Tally {
1544            first_choice,
1545            borda,
1546            winner,
1547            rankings: tops.len(),
1548            unanimous_initial,
1549            deliberated,
1550            changed_votes,
1551            unanimous_final,
1552            tie_break,
1553            judges: judges_total,
1554            present,
1555            quorum,
1556            met_quorum,
1557            uncontested,
1558        });
1559        self.state.status = if met_quorum {
1560            RunStatus::Reviewing
1561        } else {
1562            RunStatus::Stalled
1563        };
1564        self.state.save()?;
1565        Ok(())
1566    }
1567
1568    // ------------------------------------------------------------- recover
1569
1570    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
1571    /// resumed toward completion once the transient cause clears.
1572    ///
1573    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
1574    /// it toward the quorum, which is exactly the set of seats whose absence
1575    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
1576    /// not depend on which node happened to hit the limit), or an ordinary
1577    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
1578    /// seat is never disturbed.
1579    ///
1580    /// A seat that now answers with a usable ranking is "recovered": its
1581    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
1582    /// `tally` counts it present again), and its vote re-collected. A seat that
1583    /// still fails keeps its loss and stays absent.
1584    ///
1585    /// Returns `true` when the re-tally restores the quorum (the run may proceed
1586    /// to review/gate/merge), `false` when it is still below quorum (the run
1587    /// stays `Stalled`, still resumable for a later retry).
1588    #[allow(clippy::too_many_lines)]
1589    async fn recover_stall(&mut self) -> Result<bool> {
1590        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
1591        // agent files with `magi task add` name the run that paid for it. The
1592        // prompt overlay is cloned alongside it because the waves borrow it
1593        // while `self` is mutably borrowed by the node's own bookkeeping.
1594        let run_id = self.state.id.clone();
1595        let prompts = self.state.config.prompts.clone();
1596        // Absent seats = quota-lost at any node, or failed outright. Mirroring
1597        // `tally`'s presence test (rather than the old quota-judge/vote filter)
1598        // is what keeps a non-quota collapse — or a quota loss recorded at the
1599        // deliberate node — from being a permanent dead-end on `--resume`.
1600        let quota_seats: BTreeSet<&str> =
1601            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
1602        let absent: Vec<String> = self
1603            .state
1604            .judgements
1605            .iter()
1606            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
1607            .map(|j| j.seat.clone())
1608            .collect();
1609        if absent.is_empty() {
1610            return Ok(false);
1611        }
1612        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
1613        if viable.len() <= 1 {
1614            return Ok(false);
1615        }
1616        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
1617        let language = self.state.config.graph.language.clone();
1618        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
1619        let sessions = self.state.config.graph.sessions;
1620        let artifacts = agent::artifacts_dir(&self.state.dir());
1621        let root = self.state.worktree_root();
1622        let base_short = short(&self.state.base_commit);
1623        let candidates: Vec<Candidate> = viable.clone();
1624
1625        // Map each absent seat key to its 0-based position in `roles.judges`.
1626        let mut positions: Vec<usize> = absent
1627            .iter()
1628            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
1629            .collect();
1630        if positions.is_empty() {
1631            return Ok(false);
1632        }
1633        positions.sort_unstable();
1634        positions.dedup();
1635
1636        // Re-rank the lost seats, one blind prompt each.
1637        let mut judge_jobs = Vec::new();
1638        for &j in &positions {
1639            let order = blind::presentation_order(viable.len(), j, self.state.seed);
1640            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
1641            let seat_key = format!("judge-{}", j + 1);
1642            let spec = self.roles.judges[j].clone();
1643            let seat = self.seat(&seat_key, &spec.id);
1644            judge_jobs.push(SeatJob {
1645                spec,
1646                seat,
1647                prompt: prompt::judge(
1648                    &self.state.instruction,
1649                    &views,
1650                    self.roles.judges.len(),
1651                    &base_short,
1652                    &language,
1653                ),
1654                cwd: root.join(seat_key),
1655                timeout,
1656                allow_write: false,
1657                sessions,
1658                artifacts: artifacts.clone(),
1659                stem: format!("judge-{}-recover", j + 1),
1660            });
1661        }
1662
1663        let labels_for_check = labels.clone();
1664        let mut judge_losses = Vec::new();
1665        let retries = self.state.config.graph.retries;
1666        let cache = self.state.config.cache_dir();
1667        let ctx = WaveCtx {
1668            run: &run_id,
1669            node: "judge",
1670            prompts: &prompts,
1671            cache: cache.as_deref(),
1672        };
1673        let results = ask_json_wave::<Ranking>(
1674            judge_jobs,
1675            Arc::clone(&self.sem),
1676            retries,
1677            &ctx,
1678            &mut judge_losses,
1679            &mut self.state,
1680            &move |r: &Ranking| r.validate(&labels_for_check),
1681        )
1682        .await;
1683
1684        // Refresh the judgement of every seat that ranked again.
1685        let mut recovered: BTreeSet<usize> = BTreeSet::new();
1686        for (&j, (seat, res)) in positions.iter().zip(results) {
1687            self.state.seats.insert(seat.key.clone(), seat);
1688            let record = &mut self.state.judgements[j];
1689            match res {
1690                Ok((ranking, out)) => {
1691                    record.ranking = ranking.normalized();
1692                    record.reasons = ranking.reasons;
1693                    record.confidence = ranking.confidence;
1694                    record.failed = None;
1695                    record.duration_ms = out.duration_ms;
1696                    recovered.insert(j);
1697                    self.state.event(
1698                        "recover",
1699                        format!("judge {} ranked again after the limit", j + 1),
1700                    );
1701                }
1702                Err(e) => {
1703                    self.state
1704                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
1705                }
1706            }
1707        }
1708
1709        // Re-ask the votes of the seats that recovered a ranking.
1710        let mut vote_jobs = Vec::new();
1711        let mut vote_pos: Vec<usize> = Vec::new();
1712        for &j in &recovered {
1713            let seat_key = format!("judge-{}", j + 1);
1714            let spec = self.roles.judges[j].clone();
1715            let seat = self.seat(&seat_key, &spec.id);
1716            let mut text = prompt::final_vote(&labels, &language);
1717            if !has_context(&spec, &seat, sessions) {
1718                text = format!(
1719                    "{}\n\n# Candidates\n\n{}",
1720                    text,
1721                    self.candidate_block(&candidates, &base_short)
1722                );
1723            }
1724            vote_jobs.push(SeatJob {
1725                spec,
1726                seat,
1727                prompt: text,
1728                cwd: root.join(seat_key),
1729                timeout,
1730                allow_write: false,
1731                sessions,
1732                artifacts: artifacts.clone(),
1733                stem: format!("vote-judge-{}-recover", j + 1),
1734            });
1735            vote_pos.push(j);
1736        }
1737        let allowed = labels.clone();
1738        let mut vote_losses = Vec::new();
1739        let vote_retries = self.state.config.graph.retries;
1740        let vote_cache = self.state.config.cache_dir();
1741        let ctx = WaveCtx {
1742            run: &run_id,
1743            node: "vote",
1744            prompts: &prompts,
1745            cache: vote_cache.as_deref(),
1746        };
1747        let votes = ask_json_wave::<FinalVote>(
1748            vote_jobs,
1749            Arc::clone(&self.sem),
1750            vote_retries,
1751            &ctx,
1752            &mut vote_losses,
1753            &mut self.state,
1754            &move |v: &FinalVote| match v.label() {
1755                Some(c) if allowed.contains(&c) => Ok(()),
1756                other => bail!("vote {other:?} is not one of {allowed:?}"),
1757            },
1758        )
1759        .await;
1760        for (&j, (seat, res)) in vote_pos.iter().zip(votes) {
1761            let agent_id = seat.agent.clone();
1762            self.state.seats.insert(seat.key.clone(), seat);
1763            match res {
1764                Ok((v, _)) => {
1765                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
1766                        rec.vote = v.label();
1767                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
1768                    } else {
1769                        self.state.votes.push(VoteRecord {
1770                            judge: j + 1,
1771                            agent: agent_id,
1772                            vote: v.label(),
1773                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
1774                            changed: false,
1775                        });
1776                    }
1777                    self.state.event(
1778                        "recover",
1779                        format!("judge {} voted again after the limit", j + 1),
1780                    );
1781                }
1782                Err(e) => {
1783                    self.state
1784                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
1785                }
1786            }
1787        }
1788
1789        // A seat that ranked again is present even if its re-vote failed —
1790        // `tally` falls back to the initial ranking's first choice — so clear
1791        // its quota loss. Seats that still fail keep theirs and stay absent.
1792        if !recovered.is_empty() {
1793            let recovered_keys: BTreeSet<String> = recovered
1794                .iter()
1795                .map(|&j| format!("judge-{}", j + 1))
1796                .collect();
1797            self.state
1798                .quota
1799                .retain(|q| !recovered_keys.contains(&q.seat));
1800        }
1801
1802        // Recompute the verdict from the refreshed panel.
1803        self.state.tally = None;
1804        self.tally()?;
1805        Ok(self
1806            .state
1807            .tally
1808            .as_ref()
1809            .map(|t| t.met_quorum)
1810            .unwrap_or(false))
1811    }
1812
1813    // ----------------------------------------------------------------- fold
1814
1815    async fn fold_losers(&mut self) -> Result<()> {
1816        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
1817            return Ok(());
1818        };
1819        let repo = self.state.repo.clone();
1820        let mut folded = Vec::new();
1821        for i in 0..self.state.candidates.len() {
1822            let c = &self.state.candidates[i];
1823            if c.label == winner || c.folded {
1824                continue;
1825            }
1826            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
1827            git::worktree_remove(&repo, &wt).await.ok();
1828            git::branch_delete(&repo, &branch).await.ok();
1829            self.state.candidates[i].folded = true;
1830            folded.push(label.to_string());
1831        }
1832        // The judges are finished; their checkouts are pure cost from here.
1833        let root = self.state.worktree_root();
1834        for j in 1..=self.roles.judges.len() {
1835            let wt = root.join(format!("judge-{j}"));
1836            if wt.exists() {
1837                git::worktree_remove(&repo, &wt).await.ok();
1838            }
1839        }
1840        if !folded.is_empty() {
1841            self.state
1842                .event("fold", format!("folded candidates {}", folded.join(", ")));
1843            self.state.save()?;
1844        }
1845        Ok(())
1846    }
1847
1848    // ------------------------------------------------------------ base sync
1849
1850    /// Land the winner's tree on the current tip of `<remote>/<base>` before
1851    /// anything verifies it.
1852    ///
1853    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
1854    /// read whatever is checked out in the winner's worktree. Left alone that
1855    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
1856    /// it when the run *branched* - and a run takes long enough that the base
1857    /// has usually moved by the time it gets here. A gate that ran there
1858    /// answers "green on the commit this run started from", not "green on
1859    /// what is about to land", and the difference showed up three times in
1860    /// one day as a green run whose merge would have reverted a file another
1861    /// pull request had already landed.
1862    ///
1863    /// Reuses [`git::rebase_branch_in_temp`] rather than a second
1864    /// implementation of the same idea: `land::Step::Rebase` already worked
1865    /// out the rules - throwaway worktree, conflict stops and reports rather
1866    /// than feeding a fixer, nothing runs in the primary tree - and a second
1867    /// rebase path is exactly the kind of drift `resolve_base`'s own doc
1868    /// warns about ("two answers to a question nobody notices until a diff is
1869    /// wrong").
1870    ///
1871    /// Bounded by [`BASE_SYNC_ROUNDS`], counted in `state.base_sync.attempts`
1872    /// so it survives a park/resume. A conflict or a push failure sets
1873    /// `state.base_sync.conflict` and leaves the branch and worktree exactly
1874    /// as they were - untouched, for a person to look at - which is also what
1875    /// makes re-entering this function afterwards a no-op instead of a second
1876    /// attempt at the same wall.
1877    async fn sync_to_base(&mut self) -> Result<()> {
1878        if self
1879            .state
1880            .base_sync
1881            .as_ref()
1882            .is_some_and(|s| s.conflict.is_some())
1883        {
1884            return Ok(());
1885        }
1886        let Some(winner) = self.state.winner().cloned() else {
1887            return Ok(());
1888        };
1889
1890        let repo = self.state.repo.clone();
1891        let remote = self.state.config.merge.remote.clone();
1892        let base_branch = self.state.base_branch.clone();
1893        let tracking = format!("{remote}/{base_branch}");
1894
1895        git::fetch(&repo, &remote, &base_branch).await.ok();
1896        // No network, or the remote never had this branch: `resolve_base`
1897        // already treats that as non-fatal at branch time, and a run that got
1898        // this far must not be blocked by it here either.
1899        let Ok(tip) = git::rev_parse(&repo, &tracking).await else {
1900            return Ok(());
1901        };
1902
1903        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1904        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
1905        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
1906
1907        if behind == 0 {
1908            self.state.base_sync = Some(BaseSync {
1909                tip,
1910                behind: 0,
1911                attempts,
1912                conflict: None,
1913            });
1914            self.state.save()?;
1915            return Ok(());
1916        }
1917
1918        if attempts >= BASE_SYNC_ROUNDS {
1919            let why = format!(
1920                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
1921                 rebase(s); rebasing again would only race it",
1922                winner.branch
1923            );
1924            self.state.status = RunStatus::Blocked;
1925            self.state.base_sync = Some(BaseSync {
1926                tip,
1927                behind,
1928                attempts,
1929                conflict: Some(why.clone()),
1930            });
1931            self.state.event("land", why);
1932            self.state.save()?;
1933            return Ok(());
1934        }
1935
1936        self.state.event(
1937            "land",
1938            format!(
1939                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
1940                winner.branch
1941            ),
1942        );
1943        self.state.save()?;
1944
1945        let scratch = self.state.dir().join("base-sync");
1946        let rebased = git::rebase_branch_in_temp(&repo, &scratch, &winner.branch, &tracking).await;
1947        let attempts = attempts + 1;
1948        match rebased {
1949            Ok(None) => {
1950                // The branch ref moved, but a worktree that already had it
1951                // checked out (the winner's) was not told; sync its index and
1952                // files before anything reads them.
1953                git::sync_to_head(&winner.worktree).await?;
1954                self.state.base_sync = Some(BaseSync {
1955                    tip: tip.clone(),
1956                    behind: 0,
1957                    attempts,
1958                    conflict: None,
1959                });
1960                self.state
1961                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
1962            }
1963            Ok(Some(conflict)) => {
1964                let why = format!(
1965                    "{} conflicts with {tracking} and did not rebase: {}",
1966                    winner.branch,
1967                    conflict.chars().take(600).collect::<String>()
1968                );
1969                self.state.status = RunStatus::Blocked;
1970                self.state.base_sync = Some(BaseSync {
1971                    tip,
1972                    behind,
1973                    attempts,
1974                    conflict: Some(why.clone()),
1975                });
1976                self.state.event("land", why);
1977            }
1978            Err(e) => {
1979                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
1980                self.state.status = RunStatus::Blocked;
1981                self.state.base_sync = Some(BaseSync {
1982                    tip,
1983                    behind,
1984                    attempts,
1985                    conflict: Some(why.clone()),
1986                });
1987                self.state.event("land", why);
1988            }
1989        }
1990        self.state.save()?;
1991        Ok(())
1992    }
1993
1994    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
1995    /// last landed the winner on, once it has run, else the commit the run
1996    /// branched from.
1997    ///
1998    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
1999    /// and `vote` all happen before there is a winner to rebase, so they
2000    /// compare every candidate against the branch point on purpose, and a
2001    /// base that moves after they are already done cannot change an answer
2002    /// they already gave.
2003    fn landing_base(&self) -> String {
2004        self.state
2005            .base_sync
2006            .as_ref()
2007            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
2008    }
2009
2010    // --------------------------------------------------------------- review
2011
2012    async fn review_loop(&mut self) -> Result<()> {
2013        // A base that would not rebase is a person's decision, not a review
2014        // round: nothing here would change the answer, and reviewers and a
2015        // fixer would be spending real budget on a tree that cannot land
2016        // regardless of what they find.
2017        if self
2018            .state
2019            .base_sync
2020            .as_ref()
2021            .is_some_and(|s| s.conflict.is_some())
2022        {
2023            return Ok(());
2024        }
2025        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2026        // agent files with `magi task add` name the run that paid for it. The
2027        // prompt overlay is cloned alongside it because the waves borrow it
2028        // while `self` is mutably borrowed by the node's own bookkeeping.
2029        let run_id = self.state.id.clone();
2030        let prompts = self.state.config.prompts.clone();
2031        let Some(winner) = self.state.winner().cloned() else {
2032            return Ok(());
2033        };
2034        let max_rounds = self.state.config.graph.review_rounds;
2035        if max_rounds == 0 || self.state.reviews.iter().any(|r| r.clean) {
2036            self.state.status = RunStatus::Gating;
2037            self.state.save()?;
2038            return Ok(());
2039        }
2040        if self.state.reviews.len() >= max_rounds {
2041            // Every round is already spent and none was clean — this is a
2042            // reentry into a run that reached `Blocked` and stopped. The loop
2043            // below runs an empty range in that case and would otherwise fall
2044            // through without touching `status`, silently handing back
2045            // whatever an earlier node in this same walk clobbered it to (a
2046            // solo-candidate `judge`/`deliberate` skip both rewrite `status`
2047            // on every reentry). Restate the conclusion instead of leaving it
2048            // to chance.
2049            self.state.status = RunStatus::Blocked;
2050            self.state.save()?;
2051            return Ok(());
2052        }
2053        self.state.status = RunStatus::Reviewing;
2054
2055        let repo = self.state.repo.clone();
2056        let root = self.state.worktree_root();
2057        let language = self.state.config.graph.language.clone();
2058        let sessions = self.state.config.graph.sessions;
2059        let artifacts = agent::artifacts_dir(&self.state.dir());
2060        let base = self.landing_base();
2061        let base_short = short(&base);
2062        let reviewers = self.roles.reviewers.clone();
2063        let shell = self.state.config.shell();
2064
2065        let mut prev_e2e: Option<String> = None;
2066        for round in (self.state.reviews.len() + 1)..=max_rounds {
2067            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
2068            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
2069            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
2070
2071            // Each reviewer gets its own detached checkout of exactly this
2072            // commit: nobody can perturb the winner's tree, and the fixer can
2073            // keep working without racing a reviewer.
2074            let mut jobs = Vec::new();
2075            for (r, spec) in reviewers.iter().cloned().enumerate() {
2076                let wt = root.join(format!("review-{}", r + 1));
2077                if wt.exists() {
2078                    git::reset_detached(&wt, &head).await?;
2079                } else {
2080                    git::worktree_add_detached(&repo, &wt, &head).await?;
2081                }
2082                let seat_key = format!("review-{}", r + 1);
2083                let seat = self.seat(&seat_key, &spec.id);
2084                jobs.push(SeatJob {
2085                    prompt: prompt::review(&prompt::ReviewCtx {
2086                        instruction: &self.state.instruction,
2087                        branch: &winner.branch,
2088                        base_short: &base_short,
2089                        stat: &stat,
2090                        patch: &patch,
2091                        e2e: prev_e2e.as_deref(),
2092                        reviewers: reviewers.len(),
2093                        round,
2094                        rounds: max_rounds,
2095                        // A review-only run has no rankings, so nothing
2096                        // competed for this patch and the reviewer is told so.
2097                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
2098                        language: &language,
2099                    }),
2100                    spec,
2101                    seat,
2102                    cwd: wt,
2103                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
2104                    allow_write: false,
2105                    sessions,
2106                    artifacts: artifacts.clone(),
2107                    stem: format!("review-{round}-{}", r + 1),
2108                });
2109            }
2110
2111            self.state.event(
2112                "review",
2113                format!(
2114                    "round {round}: {} reviewers on {}",
2115                    jobs.len(),
2116                    short(&head)
2117                ),
2118            );
2119            let mut quota_losses = Vec::new();
2120            let review_retries = self.state.config.graph.retries;
2121            let review_cache = self.state.config.cache_dir();
2122            let ctx = WaveCtx {
2123                run: &run_id,
2124                node: "review",
2125                prompts: &prompts,
2126                cache: review_cache.as_deref(),
2127            };
2128            let results = ask_json_wave::<Review>(
2129                jobs,
2130                Arc::clone(&self.sem),
2131                review_retries,
2132                &ctx,
2133                &mut quota_losses,
2134                &mut self.state,
2135                &|_: &Review| Ok(()),
2136            )
2137            .await;
2138            self.state.quota.extend(quota_losses);
2139
2140            let mut records = Vec::new();
2141            let mut all_findings = Vec::new();
2142            for (r, (seat, res)) in results.into_iter().enumerate() {
2143                let agent_id = seat.agent.clone();
2144                self.state.seats.insert(seat.key.clone(), seat);
2145                let mut record = ReviewRecord {
2146                    reviewer: r + 1,
2147                    agent: agent_id,
2148                    summary: String::new(),
2149                    findings: Vec::new(),
2150                    failed: None,
2151                    duration_ms: 0,
2152                };
2153                match res {
2154                    Ok((review, out)) => {
2155                        record.summary = review.summary;
2156                        record.duration_ms = out.duration_ms;
2157                        for (n, mut f) in review.findings.into_iter().enumerate() {
2158                            // ids are magi's, never the agent's: the fixer's
2159                            // adoption report is keyed by them.
2160                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
2161                            all_findings.push(f.clone());
2162                            record.findings.push(f);
2163                        }
2164                        self.state.event(
2165                            "review",
2166                            format!(
2167                                "round {round}: reviewer {} raised {} finding(s)",
2168                                r + 1,
2169                                record.findings.len()
2170                            ),
2171                        );
2172                    }
2173                    Err(e) => {
2174                        record.failed = Some(e.to_string());
2175                        self.state.event(
2176                            "review",
2177                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
2178                        );
2179                    }
2180                }
2181                records.push(record);
2182            }
2183
2184            let mut e2e = run_commands(
2185                &shell,
2186                &self.state.config.verify.e2e,
2187                &winner.worktree,
2188                Duration::from_secs(self.state.config.graph.timeout_review),
2189            )
2190            .await;
2191            for o in &e2e {
2192                self.state.event(
2193                    "verify",
2194                    format!("round {round}: `{}` -> {}", o.command, e2e_outcome_label(o)),
2195                );
2196            }
2197
2198            // A build/link failure is not a verdict on the patch — it is
2199            // frequently a race against a shared `CARGO_TARGET_DIR` (see
2200            // AGENTS.md). Give verify one retry before letting a red like
2201            // that decide the round.
2202            let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
2203            if verify_retried {
2204                self.state.event(
2205                    "verify",
2206                    format!(
2207                        "round {round}: verify could not build/link, not a test result — \
2208                         retrying once before concluding"
2209                    ),
2210                );
2211                e2e = run_commands(
2212                    &shell,
2213                    &self.state.config.verify.e2e,
2214                    &winner.worktree,
2215                    Duration::from_secs(self.state.config.graph.timeout_review),
2216                )
2217                .await;
2218                for o in &e2e {
2219                    self.state.event(
2220                        "verify",
2221                        format!(
2222                            "round {round}: retry `{}` -> {}",
2223                            o.command,
2224                            e2e_outcome_label(o)
2225                        ),
2226                    );
2227                }
2228            }
2229
2230            let e2e_failures: String = e2e
2231                .iter()
2232                .filter(|o| !o.ok())
2233                .map(|o| format!("$ {}\n{}\n", o.command, o.output_tail))
2234                .collect();
2235
2236            let expected = records.len();
2237            let answered = records.iter().filter(|r| r.failed.is_none()).count();
2238            let incomplete = answered < expected;
2239            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
2240            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
2241            let policy = self.state.config.graph.incomplete_review;
2242            let clean = round_is_clean(blocking, e2e_ok, answered, expected, policy);
2243
2244            let mut round_record = ReviewRound {
2245                round,
2246                head: head.clone(),
2247                reviews: records,
2248                e2e,
2249                verify_retried,
2250                fix: None,
2251                blocking,
2252                answered,
2253                expected,
2254                clean,
2255            };
2256
2257            if incomplete {
2258                let missing: Vec<String> = round_record
2259                    .reviews
2260                    .iter()
2261                    .filter(|r| r.failed.is_some())
2262                    .map(|r| format!("review-{}", r.reviewer))
2263                    .collect();
2264                self.state.event(
2265                    "review",
2266                    format!(
2267                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
2268                        missing.join(", ")
2269                    ),
2270                );
2271            }
2272
2273            if clean {
2274                self.state.event(
2275                    "review",
2276                    if incomplete {
2277                        format!(
2278                            "round {round}: clean (warn policy, incomplete panel) — no \
2279                             blocking findings from the seats that answered, verification green"
2280                        )
2281                    } else {
2282                        format!("round {round}: clean — no blocking findings, verification green")
2283                    },
2284                );
2285                self.state.reviews.push(round_record);
2286                self.state.status = RunStatus::Gating;
2287                self.state.save()?;
2288                return Ok(());
2289            }
2290
2291            // Nothing was raised and verification passed, but not every seat
2292            // answered and the policy refuses to call that clean: re-review
2293            // rather than send the fixer after a round with nothing to fix.
2294            if incomplete && blocking == 0 && e2e_ok {
2295                self.state.reviews.push(round_record);
2296                self.state.save()?;
2297                if round == max_rounds {
2298                    self.state.status = RunStatus::Blocked;
2299                    self.state.event(
2300                        "review",
2301                        format!(
2302                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
2303                             refusing to call it clean",
2304                            expected - answered
2305                        ),
2306                    );
2307                    return Ok(());
2308                }
2309                prev_e2e = None;
2310                continue;
2311            }
2312
2313            if round == max_rounds {
2314                self.state.reviews.push(round_record);
2315                self.state.status = RunStatus::Blocked;
2316                self.state.event(
2317                    "review",
2318                    format!("{blocking} blocking finding(s) still open after {max_rounds} rounds"),
2319                );
2320                self.state.save()?;
2321                return Ok(());
2322            }
2323
2324            // Fix. The winner's own implementer seat continues its conversation:
2325            // the competition is over, so context is pure benefit now.
2326            let (fix_spec, fix_seat_key) = match &self.roles.fixer {
2327                Some(f) if f.id != winner.agent => (f.clone(), "fix".to_owned()),
2328                _ => (
2329                    self.state
2330                        .config
2331                        .agent(&winner.agent)
2332                        .cloned()
2333                        .unwrap_or_else(|_| self.roles.implementers[winner.index].clone()),
2334                    format!("impl-{}", winner.label),
2335                ),
2336            };
2337            let seat = self.seat(&fix_seat_key, &fix_spec.id);
2338            let blocking_findings: Vec<_> = all_findings
2339                .iter()
2340                .filter(|f| f.severity.blocks())
2341                .cloned()
2342                .collect();
2343            let job = SeatJob {
2344                prompt: prompt::fix(
2345                    &self.state.instruction,
2346                    &blocking_findings,
2347                    (!e2e_failures.is_empty()).then_some(e2e_failures.as_str()),
2348                    round,
2349                    max_rounds,
2350                    &language,
2351                ),
2352                spec: fix_spec.clone(),
2353                seat,
2354                cwd: winner.worktree.clone(),
2355                timeout: Duration::from_secs(self.state.config.graph.timeout_fix),
2356                allow_write: true,
2357                sessions,
2358                artifacts: artifacts.clone(),
2359                stem: format!("fix-{round}"),
2360            };
2361            let before = git::rev_parse(&winner.worktree, "HEAD").await?;
2362            let cache = self.state.config.cache_dir();
2363            let ctx = WaveCtx {
2364                run: &run_id,
2365                node: "fix",
2366                prompts: &prompts,
2367                cache: cache.as_deref(),
2368            };
2369            let (seat, out) = run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
2370            let agent_id = seat.agent.clone();
2371            let seat_key = seat.key.clone();
2372            self.state.seats.insert(seat.key.clone(), seat);
2373
2374            let mut fix = FixRecord {
2375                agent: agent_id,
2376                addressed: Vec::new(),
2377                rejected: Vec::new(),
2378                notes: String::new(),
2379                committed: false,
2380                failed: None,
2381                duration_ms: 0,
2382            };
2383            match out {
2384                AgentOutcome::Ok(o) => {
2385                    fix.duration_ms = o.duration_ms;
2386                    match verdict::extract_json::<FixReport>(&o.text) {
2387                        Ok(report) => {
2388                            fix.addressed = report.addressed;
2389                            fix.rejected = report.rejected;
2390                            fix.notes =
2391                                blind::sanitize_prose(&report.notes, &self.state.config.blind);
2392                        }
2393                        Err(e) => fix.failed = Some(format!("unparsable fix report: {e}")),
2394                    }
2395                }
2396                // The CLI's raw error JSON is not a fix report to parse.
2397                AgentOutcome::Dropped(o) => {
2398                    fix.duration_ms = o.duration_ms;
2399                    let why = o
2400                        .dropped
2401                        .as_ref()
2402                        .map(|d| d.why.as_str())
2403                        .unwrap_or("the CLI ended the stream without delivering its answer");
2404                    fix.failed = Some(format!("the CLI dropped the stream ({why})"));
2405                }
2406                AgentOutcome::Quota(o) => {
2407                    self.state.quota.push(QuotaLoss {
2408                        seat: seat_key,
2409                        node: "fix".to_owned(),
2410                        at: Timestamp::now(),
2411                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
2412                    });
2413                    fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
2414                }
2415                AgentOutcome::Failed(e) => fix.failed = Some(e),
2416            }
2417            git::commit_all(
2418                &winner.worktree,
2419                &format!("magi: review round {round} fixes (uncommitted work)"),
2420            )
2421            .await
2422            .ok();
2423            let after = git::rev_parse(&winner.worktree, "HEAD").await?;
2424            fix.committed = after != before;
2425            let commit_note = if fix.committed {
2426                "committed"
2427            } else {
2428                "NO new commit"
2429            };
2430            self.state.event(
2431                "fix",
2432                match &fix.failed {
2433                    // Distinct on purpose from "0 addressed, 0 rejected": the
2434                    // fixer's own diff still landed (blocking counts do keep
2435                    // falling round over round), only its adoption report did
2436                    // not come back, so this must never read like every
2437                    // finding was reviewed and declined.
2438                    Some(reason) => {
2439                        format!("round {round}: fixer's adoption report was lost ({reason}); {commit_note}")
2440                    }
2441                    None => format!(
2442                        "round {round}: {} addressed, {} rejected, {commit_note}",
2443                        fix.addressed.len(),
2444                        fix.rejected.len(),
2445                    ),
2446                },
2447            );
2448            let stalled = !fix.committed;
2449            round_record.fix = Some(fix);
2450            self.state.reviews.push(round_record);
2451            self.state.save()?;
2452
2453            prev_e2e = (!e2e_failures.is_empty()).then_some(e2e_failures);
2454
2455            if stalled {
2456                self.state.status = RunStatus::Blocked;
2457                self.state.event(
2458                    "review",
2459                    "the fixer produced no commit; stopping instead of looping on an unchanged tree"
2460                        .to_owned(),
2461                );
2462                self.state.save()?;
2463                return Ok(());
2464            }
2465        }
2466        Ok(())
2467    }
2468
2469    // ----------------------------------------------------------------- gate
2470
2471    async fn gate(&mut self) -> Result<()> {
2472        // Judged by the review record itself, not by `status`: a solo
2473        // candidate's `judge`/`deliberate` skip rewrites `status` on every
2474        // reentry (see `judge`), and trusting it here is exactly how a run
2475        // that exhausted its review budget got gated and merged a second
2476        // time around. The last round's `clean` flag is the actual verdict.
2477        // A base the winner could not be replayed onto is a decision, not a
2478        // round: there is no landing tree to gate. Read as its own record for
2479        // the same reason the review verdict is.
2480        if self.state.status == RunStatus::Failed
2481            || self
2482                .state
2483                .base_sync
2484                .as_ref()
2485                .is_some_and(|s| s.conflict.is_some())
2486            || self.state.reviews.last().is_some_and(|r| !r.clean)
2487        {
2488            return Ok(());
2489        }
2490        if !self.state.gate.is_empty() {
2491            return Ok(());
2492        }
2493        let Some(winner) = self.state.winner().cloned() else {
2494            return Ok(());
2495        };
2496        self.state.status = RunStatus::Gating;
2497        let shell = self.state.config.shell();
2498        let outcomes = run_commands(
2499            &shell,
2500            &self.state.config.verify.gate,
2501            &winner.worktree,
2502            Duration::from_secs(self.state.config.graph.timeout_review),
2503        )
2504        .await;
2505        for o in &outcomes {
2506            self.state.event(
2507                "gate",
2508                format!(
2509                    "`{}` -> {}",
2510                    o.command,
2511                    if o.ok() {
2512                        "pass".to_owned()
2513                    } else {
2514                        format!("FAIL ({:?})", o.code)
2515                    }
2516                ),
2517            );
2518        }
2519        let passed = outcomes.iter().all(CommandOutcome::ok);
2520        self.state.gate = outcomes;
2521        if !passed {
2522            self.state.status = RunStatus::Blocked;
2523            self.state.event("gate", "gate failed; not merging");
2524        }
2525        self.state.save()?;
2526        Ok(())
2527    }
2528
2529    // ---------------------------------------------------------------- merge
2530
2531    async fn merge(&mut self) -> Result<()> {
2532        // Same reasoning as `gate`: ask the review and gate records directly
2533        // rather than `status`, which a solo-candidate `judge`/`deliberate`
2534        // skip can rewrite on reentry to something that no longer says
2535        // `Blocked`.
2536        if self
2537            .state
2538            .base_sync
2539            .as_ref()
2540            .is_some_and(|s| s.conflict.is_some())
2541            || self.state.reviews.last().is_some_and(|r| !r.clean)
2542            || self.state.gate.iter().any(|o| !o.ok())
2543        {
2544            return Ok(());
2545        }
2546        // This node's own record, not `status`: `status == Ready` is not
2547        // unique to the harmless `MergeMode::None` path this line was
2548        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
2549        // run's PR was closed without merging — and on that run `mode` is
2550        // still `Pr`, so a reentry that fell through here would push and
2551        // open a second pull request. `self.state.merge` is set exactly once
2552        // this node (or `land`) has already produced a verdict, under every
2553        // mode, which is what "already done" actually means here.
2554        if self.state.merge.is_some() {
2555            return Ok(());
2556        }
2557        let Some(winner) = self.state.winner().cloned() else {
2558            return Ok(());
2559        };
2560        let repo = self.state.repo.clone();
2561        let base = self.state.base_branch.clone();
2562        let mode = self.state.config.merge.mode;
2563        let message = format!(
2564            "Merge magi run {} (candidate {})\n\n{}",
2565            self.state.id, winner.label, self.state.instruction
2566        );
2567
2568        let outcome = match mode {
2569            MergeMode::None => MergeOutcome {
2570                mode,
2571                ok: true,
2572                detail: format!("git -C {} merge --no-ff {}", repo.display(), winner.branch),
2573            },
2574            MergeMode::Local => {
2575                let on = git::current_branch(&repo).await?;
2576                if on.as_deref() != Some(base.as_str()) {
2577                    MergeOutcome {
2578                        mode,
2579                        ok: false,
2580                        detail: format!(
2581                            "{} has {} checked out, not the base branch {base}",
2582                            repo.display(),
2583                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
2584                        ),
2585                    }
2586                } else if !git::is_clean(&repo).await? {
2587                    MergeOutcome {
2588                        mode,
2589                        ok: false,
2590                        detail: format!("{} is dirty; refusing to merge", repo.display()),
2591                    }
2592                } else {
2593                    let out = git::merge_no_ff(&repo, &winner.branch, &message).await?;
2594                    MergeOutcome {
2595                        mode,
2596                        ok: out.ok(),
2597                        detail: if out.ok() { out.stdout } else { out.stderr },
2598                    }
2599                }
2600            }
2601            MergeMode::Pr => {
2602                let remote = self.state.config.merge.remote.clone();
2603                let pushed = git::push(&winner.worktree, &remote, &winner.branch).await?;
2604                if !pushed.ok() {
2605                    MergeOutcome {
2606                        mode,
2607                        ok: false,
2608                        detail: pushed.stderr,
2609                    }
2610                } else {
2611                    let out = gh_pr_create(&winner.worktree, &base, &winner.branch, &message).await;
2612                    match out {
2613                        Ok(url) => MergeOutcome {
2614                            mode,
2615                            ok: true,
2616                            detail: url,
2617                        },
2618                        Err(e) => MergeOutcome {
2619                            mode,
2620                            ok: false,
2621                            detail: e.to_string(),
2622                        },
2623                    }
2624                }
2625            }
2626        };
2627
2628        self.state.status = match (mode, outcome.ok) {
2629            (MergeMode::None, _) => RunStatus::Ready,
2630            (_, true) => RunStatus::Merged,
2631            (_, false) => RunStatus::Blocked,
2632        };
2633        self.state.event(
2634            "merge",
2635            format!(
2636                "{:?}: {}",
2637                mode,
2638                outcome.detail.lines().next().unwrap_or("")
2639            ),
2640        );
2641        self.state.merge = Some(outcome);
2642        self.state.save()?;
2643
2644        // The PR is open and the run would historically stop here, leaving the
2645        // operator to watch checks, feed review comments back to a fixer, and
2646        // merge. That was done by hand six times in one session before this
2647        // existed. Opt-in, because merging is the one irreversible thing magi
2648        // can do to a repository.
2649        if self.state.config.graph.land
2650            && mode == MergeMode::Pr
2651            && self.state.status == RunStatus::Merged
2652        {
2653            let url = self
2654                .state
2655                .merge
2656                .as_ref()
2657                .map(|m| m.detail.clone())
2658                .unwrap_or_default();
2659            let url = url.lines().next().unwrap_or("").trim().to_owned();
2660            if url.starts_with("http") {
2661                // A land failure is not a lost run: the work is on a branch and
2662                // the PR is open, which is exactly where a human takes over.
2663                match land::land(&mut self.state, &url).await {
2664                    Ok(pr) => {
2665                        self.state.status = match pr.state {
2666                            land::PrLifecycle::Merged => RunStatus::Merged,
2667                            _ => RunStatus::Blocked,
2668                        };
2669                    }
2670                    Err(e) => {
2671                        self.state.status = RunStatus::Blocked;
2672                        self.state.event("land", format!("gave up: {e}"));
2673                    }
2674                }
2675                self.state.save()?;
2676            }
2677        }
2678        Ok(())
2679    }
2680
2681    // -------------------------------------------------------------- helpers
2682
2683    /// Fetch or create a seat, keeping its conversation across nodes.
2684    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
2685        if let Some(existing) = self.state.seats.get(key)
2686            && existing.agent == agent
2687        {
2688            return existing.clone();
2689        }
2690        let fresh = SeatState::new(key, agent, self.state.seed);
2691        self.state.seats.insert(key.to_owned(), fresh.clone());
2692        fresh
2693    }
2694
2695    /// A candidate rendered for judging, with the leak policy applied.
2696    fn view(&self, c: &Candidate) -> CandidateView {
2697        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
2698            .unwrap_or_default();
2699        let (patch, _) = blind::sanitize_patch(
2700            &format!("candidate {} patch", c.label),
2701            &raw,
2702            &self.state.config.blind,
2703        );
2704        CandidateView {
2705            label: c.label,
2706            branch: c.branch.clone(),
2707            summary: c.summary.clone(),
2708            stat: c.stat.clone(),
2709            patch,
2710        }
2711    }
2712
2713    /// The full candidate set as prompt text, for seats with no live session.
2714    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
2715        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
2716        prompt::judge(
2717            "(see above)",
2718            &views,
2719            self.roles.judges.len(),
2720            base_short,
2721            "en",
2722        )
2723    }
2724
2725    /// Anonymised transcript for judge `self_idx`.
2726    ///
2727    /// The initial rankings are always the opening statements. Seeding them
2728    /// only when no turn had been taken yet meant every judge after the first
2729    /// argued against a single voice instead of against the actual split — the
2730    /// disagreement is the information, so it is always on the table.
2731    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
2732        let mut turns = Vec::new();
2733        for j in &self.state.judgements {
2734            if j.ranking.is_empty() {
2735                continue;
2736            }
2737            let reasons = j
2738                .reasons
2739                .iter()
2740                .map(|(k, v)| format!("- {k}: {v}"))
2741                .collect::<Vec<_>>()
2742                .join("\n");
2743            turns.push(Turn {
2744                who: format!("Judge {} (opening ranking)", j.judge),
2745                is_self: j.judge == self_idx + 1,
2746                body: format!(
2747                    "Ranked {}{}{reasons}",
2748                    j.ranking.iter().collect::<String>(),
2749                    if reasons.is_empty() {
2750                        ""
2751                    } else {
2752                        ", because:\n"
2753                    }
2754                ),
2755            });
2756        }
2757        for t in self
2758            .state
2759            .deliberation
2760            .iter()
2761            .flat_map(|r| r.turns.iter())
2762            .chain(current)
2763        {
2764            turns.push(Turn {
2765                who: format!("Judge {}", t.judge),
2766                is_self: t.judge == self_idx + 1,
2767                body: t.body.clone(),
2768            });
2769        }
2770        turns
2771    }
2772}
2773
2774/// Does this seat still hold the context a follow-up prompt would rely on?
2775fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
2776    agent::has_session(spec.kind, seat, sessions)
2777}
2778
2779fn short(commit: &str) -> String {
2780    commit.chars().take(7).collect()
2781}
2782
2783fn make_executable(path: &Path) -> Result<()> {
2784    #[cfg(unix)]
2785    {
2786        use std::os::unix::fs::PermissionsExt as _;
2787        let mut perms = std::fs::metadata(path)?.permissions();
2788        perms.set_mode(0o755);
2789        std::fs::set_permissions(path, perms)?;
2790    }
2791    #[cfg(not(unix))]
2792    {
2793        let _ = path;
2794    }
2795    Ok(())
2796}
2797
2798/// What every seat in one batch shares: where the answers are attributed, the
2799/// prompt overlay they inherit, and the build cache they are told to use.
2800///
2801/// A struct rather than four more parameters: `wave` also needs the run's
2802/// state (to record who is answering right now) and the attempt number, and
2803/// eight positional arguments is both unreadable and a clippy error.
2804struct WaveCtx<'a> {
2805    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
2806    /// paid for it.
2807    run: &'a str,
2808    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
2809    node: &'a str,
2810    prompts: &'a Prompts,
2811    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
2812    cache: Option<&'a Path>,
2813}
2814
2815/// Run one job, honouring the parallelism budget.
2816async fn run_one(
2817    job: SeatJob,
2818    sem: Arc<Semaphore>,
2819    ctx: &WaveCtx<'_>,
2820    state: &mut RunState,
2821    attempt: usize,
2822) -> (SeatState, AgentOutcome) {
2823    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
2824        .await
2825        .pop()
2826        .expect("one job in, one result out");
2827    (seat, out)
2828}
2829
2830/// Run every job concurrently, capped by the semaphore, preserving order.
2831///
2832/// Every seat in the batch is recorded into [`RunState::active`] before the
2833/// wave starts and cleared as each answer lands, so the run's own record says
2834/// who is still being waited on rather than only who finished.
2835async fn wave(
2836    jobs: Vec<SeatJob>,
2837    sem: Arc<Semaphore>,
2838    ctx: &WaveCtx<'_>,
2839    state: &mut RunState,
2840    attempt: usize,
2841) -> Vec<(usize, SeatState, AgentOutcome)> {
2842    let WaveCtx {
2843        run,
2844        node,
2845        prompts,
2846        cache,
2847    } = *ctx;
2848    for job in &jobs {
2849        state.seat_started(node, &job.seat.key, job.timeout, attempt);
2850    }
2851    if let Err(e) = state.save() {
2852        // A failed persist of "who is answering right now" must not abort the
2853        // wave: the seats are already being asked, and the alternative is
2854        // losing the answers to save a status line nobody may even be
2855        // watching.
2856        tracing::warn!("could not persist in-progress seats: {e:#}");
2857    }
2858    let mut set = tokio::task::JoinSet::new();
2859    let overlay = prompts.overlay(node);
2860    for (i, mut job) in jobs.into_iter().enumerate() {
2861        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
2862        if cache.is_some() {
2863            job.prompt.push('\n');
2864            job.prompt.push_str(prompt::build_cache_note());
2865        }
2866        let sem = Arc::clone(&sem);
2867        let run = run.to_owned();
2868        let node = node.to_owned();
2869        let cache = cache.map(Path::to_path_buf);
2870        set.spawn(async move {
2871            let _permit = sem.acquire().await;
2872            let mut seat = job.seat;
2873            let out = agent::invoke(
2874                &job.spec,
2875                &mut seat,
2876                &Invocation {
2877                    cwd: &job.cwd,
2878                    prompt: &job.prompt,
2879                    timeout: job.timeout,
2880                    allow_write: job.allow_write,
2881                    sessions: job.sessions,
2882                    artifacts: &job.artifacts,
2883                    stem: &job.stem,
2884                    run: &run,
2885                    node: &node,
2886                    cache_dir: cache.as_deref(),
2887                },
2888            )
2889            .await;
2890            let out = match out {
2891                Ok(o) if o.usable() => AgentOutcome::Ok(o),
2892                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
2893                // Billed work the CLI failed to hand over is not an ordinary
2894                // failure, but its text is the CLI's raw error JSON, not an
2895                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
2896                // read it as one by forgetting to check. `usable()` is always
2897                // false here (dropped implies an empty response), so this has
2898                // to be checked before the catch-all `Failed` below or the
2899                // one shape this exists for is lost with the rest.
2900                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
2901                Ok(o) if o.timed_out => AgentOutcome::Failed("timed out".to_owned()),
2902                Ok(o) => AgentOutcome::Failed(format!(
2903                    "exited with {:?} and no usable output",
2904                    o.exit_code
2905                )),
2906                Err(e) => AgentOutcome::Failed(e.to_string()),
2907            };
2908            (i, seat, out)
2909        });
2910    }
2911    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
2912    while let Some(joined) = set.join_next().await {
2913        let (i, seat, out) = match joined {
2914            Ok(v) => v,
2915            // No seat to clear: a panicked task never reported which one it
2916            // was. The defensive sweep below this loop is what stops that
2917            // seat's `active` entry from surviving forever.
2918            Err(e) => {
2919                tracing::error!("agent task panicked: {e}");
2920                continue;
2921            }
2922        };
2923        state.seat_finished(&seat.key);
2924        if let Err(e) = state.save() {
2925            tracing::warn!("could not persist a seat's completion: {e:#}");
2926        }
2927        if collected.len() <= i {
2928            collected.resize_with(i + 1, || None);
2929        }
2930        collected[i] = Some((i, seat, out));
2931    }
2932    // Belt-and-braces for the panic branch above: every seat this exact batch
2933    // started shares this `(node, attempt)` pair, and every seat that finished
2934    // normally already cleared itself, so anything left tagged with it here
2935    // can only be a panicked task's leftover. Cleared unconditionally rather
2936    // than left to read as still answering forever.
2937    if state
2938        .active
2939        .values()
2940        .any(|a| a.node == node && a.attempt == attempt)
2941    {
2942        state
2943            .active
2944            .retain(|_, a| !(a.node == node && a.attempt == attempt));
2945        if let Err(e) = state.save() {
2946            tracing::warn!("could not persist the end of a wave: {e:#}");
2947        }
2948    }
2949    collected.into_iter().flatten().collect()
2950}
2951
2952/// Is a review round clean, given how many reviewer seats answered against
2953/// how many the round expected?
2954///
2955/// A seat that never answered (timeout, crash, unparsable output) is not a
2956/// seat that read the patch and found nothing — treating it as such is
2957/// exactly the bug this function exists to close. Under the default `block`
2958/// policy a missing seat can never be clean; `warn` still requires the seats
2959/// that *did* answer to have found nothing blocking and verification to be
2960/// green.
2961fn round_is_clean(
2962    blocking: usize,
2963    e2e_ok: bool,
2964    answered: usize,
2965    expected: usize,
2966    policy: IncompleteReviewPolicy,
2967) -> bool {
2968    blocking == 0 && e2e_ok && (answered == expected || policy == IncompleteReviewPolicy::Warn)
2969}
2970
2971/// How long a re-ask may take, given the budget the first attempt had.
2972///
2973/// A `nudged` retry is a request to restate an answer the seat has already
2974/// worked out: it carries no new work, so it does not deserve the original
2975/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
2976/// 133 seconds while a third sat for over ten minutes on a resumed session
2977/// holding 410 KB of prior output - and because the retry had inherited the
2978/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
2979/// judging round whose other seats were long finished.
2980///
2981/// A quarter of the budget, with a floor so that a deliberately short timeout
2982/// does not collapse to nothing. A retry that re-sends the whole prompt
2983/// (because the seat kept no context) is the original job again, and keeps the
2984/// original budget.
2985fn retry_budget(full: Duration, nudged: bool) -> Duration {
2986    if nudged {
2987        (full / 4).max(Duration::from_secs(120)).min(full)
2988    } else {
2989        full
2990    }
2991}
2992
2993/// Run a wave and parse each reply, re-asking the seats whose reply was
2994/// unusable.
2995///
2996/// The re-ask is a nudge rather than the whole prompt again when the seat still
2997/// holds its conversation, which is the difference between a cheap retry and
2998/// paying for the entire candidate set twice.
2999///
3000/// A seat that hits a rate limit is **not** re-asked: the same call will fail
3001/// the same way until the limit resets, so spending a retry attempt on it is
3002/// pure waste. Its loss is recorded in `losses` and it is returned as a failure
3003/// like any other absent seat — the caller decides whether the panel still has
3004/// a quorum.
3005#[allow(clippy::too_many_arguments)]
3006async fn ask_json_wave<T>(
3007    jobs: Vec<SeatJob>,
3008    sem: Arc<Semaphore>,
3009    retries: usize,
3010    ctx: &WaveCtx<'_>,
3011    losses: &mut Vec<QuotaLoss>,
3012    state: &mut RunState,
3013    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
3014) -> Vec<(SeatState, Result<(T, AgentOutput)>)>
3015where
3016    T: serde::de::DeserializeOwned + Send + 'static,
3017{
3018    let n = jobs.len();
3019    let originals: Vec<SeatJob> = jobs;
3020    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
3021    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
3022    let mut pending: Vec<usize> = (0..n).collect();
3023
3024    for attempt in 0..=retries {
3025        if pending.is_empty() {
3026            break;
3027        }
3028        let mut batch = Vec::with_capacity(pending.len());
3029        for &i in &pending {
3030            let src = &originals[i];
3031            // The prompt and the budget are one decision: a nudge restates
3032            // finished work, a re-sent prompt redoes it.
3033            let (prompt, timeout) = if attempt == 0 {
3034                (src.prompt.clone(), src.timeout)
3035            } else {
3036                let why = done[i]
3037                    .as_ref()
3038                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
3039                    .unwrap_or_else(|| "no parsable answer".to_owned());
3040                let nudge = prompt::nudge(&why);
3041                let nudged = has_context(&src.spec, &seats[i], src.sessions);
3042                let prompt = if nudged {
3043                    nudge
3044                } else {
3045                    format!("{}\n\n---\n\n{}", src.prompt, nudge)
3046                };
3047                (prompt, retry_budget(src.timeout, nudged))
3048            };
3049            batch.push(SeatJob {
3050                spec: src.spec.clone(),
3051                seat: seats[i].clone(),
3052                cwd: src.cwd.clone(),
3053                prompt,
3054                timeout,
3055                allow_write: src.allow_write,
3056                sessions: src.sessions,
3057                artifacts: src.artifacts.clone(),
3058                stem: if attempt == 0 {
3059                    src.stem.clone()
3060                } else {
3061                    format!("{}-retry{attempt}", src.stem)
3062                },
3063            });
3064        }
3065
3066        if attempt > 0 {
3067            let seats_out: Vec<&str> = pending
3068                .iter()
3069                .map(|&i| originals[i].seat.key.as_str())
3070                .collect();
3071            state.event(
3072                ctx.node,
3073                format!("retry {attempt}: re-asking {}", seats_out.join(", ")),
3074            );
3075        }
3076        let results = wave(batch, Arc::clone(&sem), ctx, state, attempt).await;
3077        let mut still = Vec::new();
3078        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
3079            seats[i] = seat;
3080            let (parsed, quota) = match out {
3081                AgentOutcome::Ok(o) => (
3082                    match verdict::extract_json::<T>(&o.text) {
3083                        Ok(v) => match validate(&v) {
3084                            Ok(()) => Ok((v, o)),
3085                            Err(e) => Err(e),
3086                        },
3087                        Err(e) => Err(e),
3088                    },
3089                    false,
3090                ),
3091                AgentOutcome::Quota(o) => {
3092                    losses.push(QuotaLoss {
3093                        seat: originals[i].seat.key.clone(),
3094                        node: ctx.node.to_owned(),
3095                        at: Timestamp::now(),
3096                        reset: o.quota.as_ref().and_then(|q| q.reset.clone()),
3097                    });
3098                    (
3099                        Err(anyhow::anyhow!("rate limited (quota); not retrying now")),
3100                        true,
3101                    )
3102                }
3103                // Not a parseable answer, but also not worth a special-cased
3104                // retry here: the nudge loop above already re-asks anything
3105                // that fails to parse, which is exactly what a dropped stream
3106                // needs. Just don't hand its raw error JSON to `extract_json`.
3107                AgentOutcome::Dropped(o) => {
3108                    let why = o
3109                        .dropped
3110                        .as_ref()
3111                        .map(|d| d.why.as_str())
3112                        .unwrap_or("the CLI ended the stream without delivering its answer");
3113                    (
3114                        Err(anyhow::anyhow!("the CLI dropped the stream ({why})")),
3115                        false,
3116                    )
3117                }
3118                AgentOutcome::Failed(e) => (Err(anyhow::anyhow!(e)), false),
3119            };
3120            let failed = parsed.is_err();
3121            done[i] = Some(parsed);
3122            // Do not re-ask a rate-limited seat (quota) — a retry is known to
3123            // fail the same way; and never re-ask a seat that already parsed.
3124            if failed && !quota {
3125                still.push(i);
3126            }
3127        }
3128        pending = still;
3129    }
3130
3131    seats
3132        .into_iter()
3133        .zip(done)
3134        .map(|(seat, res)| {
3135            (
3136                seat,
3137                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
3138            )
3139        })
3140        .collect()
3141}
3142
3143/// Describe one verify command's outcome for the event log, distinguishing a
3144/// build/link failure — the toolchain never produced a binary to run — from
3145/// an actual test failure, since only the latter is a verdict on the patch.
3146fn e2e_outcome_label(o: &CommandOutcome) -> String {
3147    if o.ok() {
3148        "pass".to_owned()
3149    } else if o.build_failed() {
3150        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
3151    } else {
3152        format!("FAIL ({:?})", o.code)
3153    }
3154}
3155
3156/// Run configured shell commands in `cwd`, in order.
3157async fn run_commands(
3158    shell: &[String],
3159    commands: &[String],
3160    cwd: &Path,
3161    timeout: Duration,
3162) -> Vec<CommandOutcome> {
3163    let mut out = Vec::new();
3164    for command in commands {
3165        let started = Instant::now();
3166        let mut cmd = tokio::process::Command::new(&shell[0]);
3167        cmd.quiet();
3168        cmd.args(&shell[1..])
3169            .arg(command)
3170            .current_dir(cwd)
3171            .stdin(std::process::Stdio::null())
3172            .stdout(std::process::Stdio::piped())
3173            .stderr(std::process::Stdio::piped())
3174            .kill_on_drop(true);
3175        let spawned = cmd.spawn();
3176        let (code, body) = match spawned {
3177            Ok(child) => match tokio::time::timeout(timeout, child.wait_with_output()).await {
3178                Ok(Ok(o)) => {
3179                    let mut body = String::from_utf8_lossy(&o.stdout).into_owned();
3180                    body.push_str(&String::from_utf8_lossy(&o.stderr));
3181                    (o.status.code(), body)
3182                }
3183                Ok(Err(e)) => (None, format!("failed to run: {e}")),
3184                Err(_) => (None, format!("timed out after {}s", timeout.as_secs())),
3185            },
3186            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
3187        };
3188        out.push(CommandOutcome {
3189            command: command.clone(),
3190            code,
3191            output_tail: tail(&body, OUTPUT_TAIL),
3192            duration_ms: started.elapsed().as_millis() as u64,
3193        });
3194    }
3195    out
3196}
3197
3198/// `gh pr create`, returning the PR url.
3199async fn gh_pr_create(cwd: &Path, base: &str, head: &str, body: &str) -> Result<String> {
3200    let title = body.lines().next().unwrap_or("magi run").to_owned();
3201    let out = tokio::process::Command::new("gh")
3202        .args([
3203            "pr", "create", "--base", base, "--head", head, "--title", &title, "--body", body,
3204        ])
3205        .current_dir(cwd)
3206        .stdin(std::process::Stdio::null())
3207        .output()
3208        .await
3209        .context("spawn gh")?;
3210    if out.status.success() {
3211        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
3212    } else {
3213        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
3214    }
3215}
3216
3217/// Tear a run's worktrees and branches down.
3218pub async fn fold_run(state: &mut RunState, drop_winner: bool) -> Result<Vec<String>> {
3219    let repo = state.repo.clone();
3220    let root = state.worktree_root();
3221    let winner = state.tally.as_ref().map(|t| t.winner);
3222    let mut removed = Vec::new();
3223
3224    for i in 0..state.candidates.len() {
3225        let c = state.candidates[i].clone();
3226        let is_winner = Some(c.label) == winner;
3227        if is_winner && !drop_winner {
3228            continue;
3229        }
3230        if c.worktree.exists() {
3231            git::worktree_remove(&repo, &c.worktree).await.ok();
3232            removed.push(c.worktree.to_string_lossy().into_owned());
3233        }
3234        if git::branch_exists(&repo, &c.branch).await.unwrap_or(false) {
3235            git::branch_delete(&repo, &c.branch).await.ok();
3236            removed.push(c.branch.clone());
3237        }
3238        state.candidates[i].folded = true;
3239    }
3240
3241    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
3242        let path = name.path();
3243        let keep = !drop_winner
3244            && winner.is_some_and(|w| {
3245                path.file_name()
3246                    .is_some_and(|n| n == format!("cand-{w}").as_str())
3247            });
3248        if keep {
3249            continue;
3250        }
3251        git::worktree_remove(&repo, &path).await.ok();
3252        removed.push(path.to_string_lossy().into_owned());
3253    }
3254
3255    if state.enabled_worktree_config && drop_winner {
3256        git::disable_worktree_config(&repo).await.ok();
3257        state.enabled_worktree_config = false;
3258    }
3259    state.save()?;
3260    Ok(removed)
3261}
3262
3263/// Severity of the worst open finding in the last review round, for reporting.
3264pub fn worst_open(state: &RunState) -> Option<Severity> {
3265    state
3266        .reviews
3267        .last()?
3268        .reviews
3269        .iter()
3270        .flat_map(|r| r.findings.iter())
3271        .map(|f| f.severity)
3272        .max()
3273}
3274
3275#[cfg(test)]
3276mod tests {
3277    use super::*;
3278    use std::time::Duration;
3279
3280    // `round_is_clean` is the exact decision this task fixed: a round with a
3281    // seat that never answered must not read the same as a round every seat
3282    // actually reviewed. These are deterministic and process-free by design —
3283    // the equivalent end-to-end check (a real reviewer timing out under a
3284    // live graph run) is a genuine race against wall-clock contention, and a
3285    // spawn slow enough to blow even a generous budget under a loaded test
3286    // run must not turn this specific regression check flaky.
3287
3288    #[test]
3289    fn a_full_panel_that_found_nothing_is_clean() {
3290        assert!(round_is_clean(0, true, 2, 2, IncompleteReviewPolicy::Block));
3291    }
3292
3293    #[test]
3294    fn a_missing_seat_is_never_clean_under_the_default_policy() {
3295        assert!(!round_is_clean(
3296            0,
3297            true,
3298            1,
3299            2,
3300            IncompleteReviewPolicy::Block
3301        ));
3302    }
3303
3304    #[test]
3305    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
3306        assert!(!round_is_clean(1, true, 1, 2, IncompleteReviewPolicy::Warn));
3307    }
3308
3309    #[test]
3310    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
3311        assert!(round_is_clean(0, true, 1, 2, IncompleteReviewPolicy::Warn));
3312    }
3313
3314    #[test]
3315    fn a_full_panel_with_an_open_finding_is_not_clean() {
3316        assert!(!round_is_clean(
3317            1,
3318            true,
3319            2,
3320            2,
3321            IncompleteReviewPolicy::Block
3322        ));
3323    }
3324
3325    #[test]
3326    fn a_full_panel_with_a_red_e2e_is_not_clean() {
3327        assert!(!round_is_clean(
3328            0,
3329            false,
3330            2,
3331            2,
3332            IncompleteReviewPolicy::Block
3333        ));
3334    }
3335
3336    fn secs(n: u64) -> Duration {
3337        Duration::from_secs(n)
3338    }
3339
3340    /// A throwaway repo with one commit on `main`, for tests that need `merge`
3341    /// to make real (and, if it runs at all, real*ly fail*) git calls.
3342    fn init_repo(dir: &Path) {
3343        let run = |args: &[&str]| {
3344            let out = std::process::Command::new("git")
3345                .args(args)
3346                .current_dir(dir)
3347                .output()
3348                .expect("spawn git");
3349            assert!(
3350                out.status.success(),
3351                "git {args:?} failed: {}",
3352                String::from_utf8_lossy(&out.stderr)
3353            );
3354        };
3355        run(&["init", "-b", "main"]);
3356        run(&["config", "user.name", "magi test"]);
3357        run(&["config", "user.email", "magi@example.com"]);
3358        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
3359        run(&["add", "-A"]);
3360        run(&["commit", "-m", "init"]);
3361    }
3362
3363    /// `status == Ready` used to be read as "this is the harmless
3364    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
3365    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
3366    /// run's PR was closed without merging — and reentering `merge` with
3367    /// `mode` still `Pr` does not know the difference, so it pushed and
3368    /// opened a second pull request. `mode == Local` reproduces the same
3369    /// blind spot without a network call: reentry must not attempt another
3370    /// git merge once this node has already recorded an outcome.
3371    #[tokio::test]
3372    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
3373        let tmp = tempfile::tempdir().expect("tempdir");
3374        let repo = tmp.path().join("repo");
3375        std::fs::create_dir_all(&repo).unwrap();
3376        init_repo(&repo);
3377
3378        let mut config = Config::default();
3379        config.merge.mode = MergeMode::Local;
3380
3381        let mut state = RunState::new(
3382            repo.clone(),
3383            "main".to_owned(),
3384            "deadbeef".to_owned(),
3385            "task".to_owned(),
3386            config,
3387        );
3388        state.candidates = vec![Candidate {
3389            index: 0,
3390            label: 'A',
3391            agent: "alpha".to_owned(),
3392            branch: "does-not-exist".to_owned(),
3393            worktree: repo.clone(),
3394            summary: String::new(),
3395            stat: String::new(),
3396            files: 0,
3397            commits: 0,
3398            empty: false,
3399            failed: None,
3400            duration_ms: 0,
3401            folded: false,
3402        }];
3403        state.tally = Some(Tally {
3404            first_choice: BTreeMap::from([('A', 1)]),
3405            borda: BTreeMap::new(),
3406            winner: 'A',
3407            rankings: 1,
3408            unanimous_initial: true,
3409            deliberated: false,
3410            changed_votes: 0,
3411            unanimous_final: true,
3412            tie_break: None,
3413            judges: 0,
3414            present: 0,
3415            quorum: 0,
3416            met_quorum: true,
3417            uncontested: Some("only candidate A produced a change".to_owned()),
3418        });
3419        state.reviews = vec![ReviewRound {
3420            round: 1,
3421            head: "deadbeef".to_owned(),
3422            reviews: Vec::new(),
3423            e2e: Vec::new(),
3424            fix: None,
3425            blocking: 0,
3426            answered: 0,
3427            expected: 0,
3428            clean: true,
3429            verify_retried: false,
3430        }];
3431        state.gate = vec![CommandOutcome {
3432            command: "test".to_owned(),
3433            code: Some(0),
3434            output_tail: String::new(),
3435            duration_ms: 0,
3436        }];
3437        // Reached its conclusion already — e.g. `land` closing the PR without
3438        // merging it, which (like the honest `MergeMode::None` path) leaves
3439        // `status` at `Ready`. The recorded outcome is what actually marks
3440        // this node done.
3441        state.status = RunStatus::Ready;
3442        state.merge = Some(MergeOutcome {
3443            mode: MergeMode::Local,
3444            ok: false,
3445            detail: "already concluded".to_owned(),
3446        });
3447
3448        let mut runner = Runner {
3449            state,
3450            roles: ResolvedRoles {
3451                implementers: Vec::new(),
3452                judges: Vec::new(),
3453                reviewers: Vec::new(),
3454                fixer: None,
3455            },
3456            sem: Arc::new(Semaphore::new(1)),
3457            pause: Pause::new(),
3458        };
3459
3460        runner.merge().await.expect("merge");
3461
3462        assert_eq!(
3463            runner.state.status,
3464            RunStatus::Ready,
3465            "a concluded run's status must not change on reentry"
3466        );
3467        assert_eq!(
3468            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
3469            Some("already concluded"),
3470            "merge must not run again once the node already recorded an outcome"
3471        );
3472    }
3473
3474    #[test]
3475    fn a_nudge_gets_a_quarter_of_the_budget() {
3476        // The judge and implement budgets magi ships with.
3477        assert_eq!(retry_budget(secs(1200), true), secs(300));
3478        assert_eq!(retry_budget(secs(3600), true), secs(900));
3479    }
3480
3481    #[test]
3482    fn a_resent_prompt_keeps_the_whole_budget() {
3483        // The seat kept no context, so the retry is the original job again and
3484        // shortening it would only guarantee a second failure.
3485        assert_eq!(retry_budget(secs(1200), false), secs(1200));
3486        assert_eq!(retry_budget(secs(60), false), secs(60));
3487    }
3488
3489    #[test]
3490    fn the_floor_never_exceeds_the_original_budget() {
3491        // A short configured timeout must not be *raised* by the floor: the
3492        // operator asked for a bound, and a retry may not outlast the attempt
3493        // it is retrying.
3494        assert_eq!(retry_budget(secs(60), true), secs(60));
3495        assert_eq!(retry_budget(secs(480), true), secs(120));
3496        assert_eq!(retry_budget(secs(0), true), secs(0));
3497    }
3498}